2024-11-11 02:01:44,391 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-11 02:01:44,407 main DEBUG Took 0.014119 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 02:01:44,408 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 02:01:44,408 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 02:01:44,410 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 02:01:44,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,420 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 02:01:44,434 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,438 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,440 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,441 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,443 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,444 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,445 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,446 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,448 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,449 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 02:01:44,450 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,450 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 02:01:44,452 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 02:01:44,454 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 02:01:44,457 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 02:01:44,457 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 02:01:44,459 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 02:01:44,460 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 02:01:44,470 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 02:01:44,474 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 02:01:44,476 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 02:01:44,477 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 02:01:44,477 main DEBUG createAppenders(={Console}) 2024-11-11 02:01:44,478 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-11 02:01:44,479 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-11 02:01:44,479 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-11 02:01:44,480 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 02:01:44,480 main DEBUG OutputStream closed 2024-11-11 02:01:44,481 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 02:01:44,481 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 02:01:44,481 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-11 02:01:44,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 02:01:44,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 02:01:44,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 02:01:44,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 02:01:44,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 02:01:44,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 02:01:44,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 02:01:44,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 02:01:44,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 02:01:44,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 02:01:44,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 02:01:44,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 02:01:44,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 02:01:44,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 02:01:44,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 02:01:44,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 02:01:44,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 02:01:44,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 02:01:44,592 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 02:01:44,593 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-11 02:01:44,593 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 02:01:44,594 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-11T02:01:44,828 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865 2024-11-11 02:01:44,831 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 02:01:44,831 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T02:01:44,839 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncLogRolling timeout: 13 mins 2024-11-11T02:01:44,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=22, ProcessCount=11, AvailableMemoryMB=7073 2024-11-11T02:01:44,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:01:44,897 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc, deleteOnExit=true 2024-11-11T02:01:44,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:01:44,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/test.cache.data in system properties and HBase conf 2024-11-11T02:01:44,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:01:44,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:01:44,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:01:44,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:01:44,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:01:45,010 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T02:01:45,096 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:01:45,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:01:45,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:01:45,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:01:45,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:01:45,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:01:45,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:01:45,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:01:45,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:01:45,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:01:45,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:01:45,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:01:45,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:01:45,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:01:45,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:01:45,838 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T02:01:45,908 INFO [Time-limited test {}] log.Log(170): Logging initialized @2223ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T02:01:45,972 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:01:46,028 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:01:46,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:01:46,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:01:46,048 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:01:46,059 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:01:46,062 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20abc9e4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:01:46,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6932e96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:01:46,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@612e4b2b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/java.io.tmpdir/jetty-localhost-44595-hadoop-hdfs-3_4_1-tests_jar-_-any-6641687247778542554/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:01:46,243 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@65270512{HTTP/1.1, (http/1.1)}{localhost:44595} 2024-11-11T02:01:46,243 INFO [Time-limited test {}] server.Server(415): Started @2559ms 2024-11-11T02:01:46,553 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:01:46,560 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:01:46,561 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:01:46,561 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:01:46,561 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:01:46,562 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2973dc0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:01:46,563 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49b96e54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:01:46,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@790fbcfe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/java.io.tmpdir/jetty-localhost-40575-hadoop-hdfs-3_4_1-tests_jar-_-any-12708442559066740821/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:01:46,664 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e3c8436{HTTP/1.1, (http/1.1)}{localhost:40575} 2024-11-11T02:01:46,665 INFO [Time-limited test {}] server.Server(415): Started @2980ms 2024-11-11T02:01:46,716 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:01:46,818 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:01:46,826 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:01:46,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:01:46,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:01:46,829 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:01:46,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d434c3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:01:46,835 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@775a6b37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:01:46,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b9fe80c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/java.io.tmpdir/jetty-localhost-34333-hadoop-hdfs-3_4_1-tests_jar-_-any-6370797806170650601/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:01:46,937 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@431c6310{HTTP/1.1, (http/1.1)}{localhost:34333} 2024-11-11T02:01:46,937 INFO [Time-limited test {}] server.Server(415): Started @3253ms 2024-11-11T02:01:46,940 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:01:47,063 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data1/current/BP-1855330386-172.17.0.2-1731290505625/current, will proceed with Du for space computation calculation, 2024-11-11T02:01:47,063 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data3/current/BP-1855330386-172.17.0.2-1731290505625/current, will proceed with Du for space computation calculation, 2024-11-11T02:01:47,063 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data2/current/BP-1855330386-172.17.0.2-1731290505625/current, will proceed with Du for space computation calculation, 2024-11-11T02:01:47,063 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data4/current/BP-1855330386-172.17.0.2-1731290505625/current, will proceed with Du for space computation calculation, 2024-11-11T02:01:47,110 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:01:47,111 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:01:47,166 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x22717afc1641bd0e with lease ID 0x62497d0cb75e9d3d: Processing first storage report for DS-b7e9aabc-904e-41d3-bee4-06d66c95067d from datanode DatanodeRegistration(127.0.0.1:39139, datanodeUuid=5521b6ce-b6a0-472f-b601-0cc73c253eca, infoPort=39855, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625) 2024-11-11T02:01:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x22717afc1641bd0e with lease ID 0x62497d0cb75e9d3d: from storage DS-b7e9aabc-904e-41d3-bee4-06d66c95067d node DatanodeRegistration(127.0.0.1:39139, datanodeUuid=5521b6ce-b6a0-472f-b601-0cc73c253eca, infoPort=39855, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T02:01:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2fc994caa597568f with lease ID 0x62497d0cb75e9d3e: Processing first storage report for DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c from datanode DatanodeRegistration(127.0.0.1:33853, datanodeUuid=80ff741e-d986-4785-bc90-082a3f73dfea, infoPort=35049, infoSecurePort=0, ipcPort=38883, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625) 2024-11-11T02:01:47,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2fc994caa597568f with lease ID 0x62497d0cb75e9d3e: from storage DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c node DatanodeRegistration(127.0.0.1:33853, datanodeUuid=80ff741e-d986-4785-bc90-082a3f73dfea, infoPort=35049, infoSecurePort=0, ipcPort=38883, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:01:47,169 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x22717afc1641bd0e with lease ID 0x62497d0cb75e9d3d: Processing first storage report for DS-176ce7f9-1eed-4351-a563-55e346e6b6fd from datanode DatanodeRegistration(127.0.0.1:39139, datanodeUuid=5521b6ce-b6a0-472f-b601-0cc73c253eca, infoPort=39855, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625) 2024-11-11T02:01:47,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x22717afc1641bd0e with lease ID 0x62497d0cb75e9d3d: from storage DS-176ce7f9-1eed-4351-a563-55e346e6b6fd node DatanodeRegistration(127.0.0.1:39139, datanodeUuid=5521b6ce-b6a0-472f-b601-0cc73c253eca, infoPort=39855, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:01:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2fc994caa597568f with lease ID 0x62497d0cb75e9d3e: Processing first storage report for DS-966868a6-28da-4bd6-90b3-a9bf4d933906 from datanode DatanodeRegistration(127.0.0.1:33853, datanodeUuid=80ff741e-d986-4785-bc90-082a3f73dfea, infoPort=35049, infoSecurePort=0, ipcPort=38883, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625) 2024-11-11T02:01:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2fc994caa597568f with lease ID 0x62497d0cb75e9d3e: from storage DS-966868a6-28da-4bd6-90b3-a9bf4d933906 node DatanodeRegistration(127.0.0.1:33853, datanodeUuid=80ff741e-d986-4785-bc90-082a3f73dfea, infoPort=35049, infoSecurePort=0, ipcPort=38883, storageInfo=lv=-57;cid=testClusterID;nsid=573794671;c=1731290505625), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:01:47,257 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865 2024-11-11T02:01:47,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/zookeeper_0, clientPort=57806, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:01:47,334 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57806 2024-11-11T02:01:47,344 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:47,347 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:47,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:01:47,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:01:47,994 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571 with version=8 2024-11-11T02:01:47,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase-staging 2024-11-11T02:01:48,084 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T02:01:48,276 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:01:48,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,289 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:01:48,289 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,289 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:01:48,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:01:48,456 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T02:01:48,465 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T02:01:48,468 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:01:48,490 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 25260 (auto-detected) 2024-11-11T02:01:48,491 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T02:01:48,508 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35677 2024-11-11T02:01:48,526 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35677 connecting to ZooKeeper ensemble=127.0.0.1:57806 2024-11-11T02:01:48,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356770x0, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:01:48,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35677-0x1002c84049b0000 connected 2024-11-11T02:01:48,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:48,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:48,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:01:48,597 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571, hbase.cluster.distributed=false 2024-11-11T02:01:48,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:01:48,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35677 2024-11-11T02:01:48,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35677 2024-11-11T02:01:48,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35677 2024-11-11T02:01:48,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35677 2024-11-11T02:01:48,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35677 2024-11-11T02:01:48,717 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:01:48,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,719 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:01:48,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:01:48,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:01:48,722 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:01:48,724 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:01:48,725 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35353 2024-11-11T02:01:48,727 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35353 connecting to ZooKeeper ensemble=127.0.0.1:57806 2024-11-11T02:01:48,728 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:48,732 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:48,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353530x0, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:01:48,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35353-0x1002c84049b0001 connected 2024-11-11T02:01:48,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:01:48,744 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:01:48,751 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:01:48,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:01:48,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:01:48,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35353 2024-11-11T02:01:48,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35353 2024-11-11T02:01:48,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35353 2024-11-11T02:01:48,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35353 2024-11-11T02:01:48,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35353 2024-11-11T02:01:48,775 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:35677 2024-11-11T02:01:48,776 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,35677,1731290508127 2024-11-11T02:01:48,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:01:48,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:01:48,785 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,35677,1731290508127 2024-11-11T02:01:48,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:01:48,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:48,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:48,804 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:01:48,805 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,35677,1731290508127 from backup master directory 2024-11-11T02:01:48,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,35677,1731290508127 2024-11-11T02:01:48,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:01:48,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:01:48,809 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:01:48,809 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,35677,1731290508127 2024-11-11T02:01:48,811 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T02:01:48,812 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T02:01:48,863 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase.id] with ID: 59902562-b714-4f7a-b65d-ec9f146b03a9 2024-11-11T02:01:48,863 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/.tmp/hbase.id 2024-11-11T02:01:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:01:48,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:01:48,876 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/.tmp/hbase.id]:[hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase.id] 2024-11-11T02:01:48,919 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:48,924 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:01:48,942 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-11T02:01:48,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:48,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:48,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:01:48,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:01:48,975 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:01:48,977 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:01:48,983 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:01:49,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:01:49,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:01:49,027 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store 2024-11-11T02:01:49,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:01:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:01:49,050 INFO [master/370bc2ade342:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T02:01:49,054 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:49,056 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:01:49,056 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:01:49,056 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:01:49,058 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:01:49,058 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:01:49,059 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:01:49,060 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290509055Disabling compacts and flushes for region at 1731290509055Disabling writes for close at 1731290509058 (+3 ms)Writing region close event to WAL at 1731290509059 (+1 ms)Closed at 1731290509059 2024-11-11T02:01:49,063 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/.initializing 2024-11-11T02:01:49,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/WALs/370bc2ade342,35677,1731290508127 2024-11-11T02:01:49,072 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:01:49,086 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C35677%2C1731290508127, suffix=, logDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/WALs/370bc2ade342,35677,1731290508127, archiveDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/oldWALs, maxLogs=10 2024-11-11T02:01:49,119 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/WALs/370bc2ade342,35677,1731290508127/370bc2ade342%2C35677%2C1731290508127.1731290509091, exclude list is [], retry=0 2024-11-11T02:01:49,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:414) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:473) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:468) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:01:49,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:01:49,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:01:49,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T02:01:49,175 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/WALs/370bc2ade342,35677,1731290508127/370bc2ade342%2C35677%2C1731290508127.1731290509091 2024-11-11T02:01:49,176 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:01:49,177 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:01:49,178 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:49,180 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,181 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:01:49,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:49,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:01:49,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:01:49,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:01:49,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:01:49,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:01:49,259 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:01:49,260 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,264 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,265 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,270 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,270 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,273 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:01:49,277 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:01:49,282 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:01:49,284 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=827320, jitterRate=0.05199195444583893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:01:49,293 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731290509192Initializing all the Stores at 1731290509194 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290509195 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290509195Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290509196 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290509196Cleaning up temporary data from old regions at 1731290509270 (+74 ms)Region opened successfully at 1731290509292 (+22 ms) 2024-11-11T02:01:49,294 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:01:49,328 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c5a88bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:01:49,355 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:01:49,364 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:01:49,364 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:01:49,367 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:01:49,369 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T02:01:49,373 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T02:01:49,373 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:01:49,402 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:01:49,413 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:01:49,415 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:01:49,418 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:01:49,420 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:01:49,421 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:01:49,423 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:01:49,426 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:01:49,427 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:01:49,429 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:01:49,430 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:01:49,446 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:01:49,447 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:01:49,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:01:49,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:01:49,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,453 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,35677,1731290508127, sessionid=0x1002c84049b0000, setting cluster-up flag (Was=false) 2024-11-11T02:01:49,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,469 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:01:49,471 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,35677,1731290508127 2024-11-11T02:01:49,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:49,480 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:01:49,482 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,35677,1731290508127 2024-11-11T02:01:49,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:01:49,559 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:01:49,566 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(746): ClusterId : 59902562-b714-4f7a-b65d-ec9f146b03a9 2024-11-11T02:01:49,568 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:01:49,569 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:01:49,573 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:01:49,573 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:01:49,576 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:01:49,576 DEBUG [RS:0;370bc2ade342:35353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec9bd52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:01:49,577 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:01:49,584 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,35677,1731290508127 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:01:49,591 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:35353 2024-11-11T02:01:49,592 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:01:49,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:01:49,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:01:49,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:01:49,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:01:49,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,594 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:01:49,594 INFO [RS:0;370bc2ade342:35353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:01:49,594 INFO [RS:0;370bc2ade342:35353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:01:49,594 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,594 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:01:49,596 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,35677,1731290508127 with port=35353, startcode=1731290508685 2024-11-11T02:01:49,597 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731290539597 2024-11-11T02:01:49,599 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:01:49,599 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:01:49,599 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:01:49,601 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:01:49,604 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:01:49,604 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:01:49,604 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:01:49,605 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:01:49,606 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,605 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,606 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:01:49,608 DEBUG [RS:0;370bc2ade342:35353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:01:49,609 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:01:49,610 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:01:49,610 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:01:49,612 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:01:49,613 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:01:49,615 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290509614,5,FailOnTimeoutGroup] 2024-11-11T02:01:49,616 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290509615,5,FailOnTimeoutGroup] 2024-11-11T02:01:49,616 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,616 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:01:49,617 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,618 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:01:49,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:01:49,623 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:01:49,624 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571 2024-11-11T02:01:49,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:01:49,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:01:49,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:49,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:01:49,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:01:49,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:01:49,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:49,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:01:49,653 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35677 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,654 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:01:49,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:49,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:01:49,656 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35677 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:01:49,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:49,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:01:49,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:01:49,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:49,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:49,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:01:49,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740 2024-11-11T02:01:49,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740 2024-11-11T02:01:49,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:01:49,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:01:49,671 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:01:49,672 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571 2024-11-11T02:01:49,672 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35985 2024-11-11T02:01:49,672 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:01:49,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:01:49,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:01:49,677 DEBUG [RS:0;370bc2ade342:35353 {}] zookeeper.ZKUtil(111): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,677 WARN [RS:0;370bc2ade342:35353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:01:49,677 INFO [RS:0;370bc2ade342:35353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:01:49,678 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,680 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:01:49,680 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,35353,1731290508685] 2024-11-11T02:01:49,681 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=699600, jitterRate=-0.11041288077831268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:01:49,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731290509644Initializing all the Stores at 1731290509646 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290509646Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290509647 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290509647Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290509647Cleaning up temporary data from old regions at 1731290509670 (+23 ms)Region opened successfully at 1731290509685 (+15 ms) 2024-11-11T02:01:49,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:01:49,685 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:01:49,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:01:49,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:01:49,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:01:49,690 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:01:49,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290509685Disabling compacts and flushes for region at 1731290509685Disabling writes for close at 1731290509686 (+1 ms)Writing region close event to WAL at 1731290509689 (+3 ms)Closed at 1731290509690 (+1 ms) 2024-11-11T02:01:49,693 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:01:49,693 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:01:49,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:01:49,707 INFO [RS:0;370bc2ade342:35353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:01:49,708 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:01:49,712 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:01:49,724 INFO [RS:0;370bc2ade342:35353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:01:49,729 INFO [RS:0;370bc2ade342:35353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:01:49,729 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,730 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:01:49,736 INFO [RS:0;370bc2ade342:35353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:01:49,737 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,738 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:01:49,739 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:01:49,740 DEBUG [RS:0;370bc2ade342:35353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:01:49,741 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,741 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,741 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,741 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,741 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,742 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35353,1731290508685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:01:49,767 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:01:49,770 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35353,1731290508685-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,770 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,770 INFO [RS:0;370bc2ade342:35353 {}] regionserver.Replication(171): 370bc2ade342,35353,1731290508685 started 2024-11-11T02:01:49,789 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:49,789 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,35353,1731290508685, RpcServer on 370bc2ade342/172.17.0.2:35353, sessionid=0x1002c84049b0001 2024-11-11T02:01:49,790 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:01:49,790 DEBUG [RS:0;370bc2ade342:35353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,790 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,35353,1731290508685' 2024-11-11T02:01:49,791 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:01:49,792 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:01:49,792 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:01:49,792 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:01:49,793 DEBUG [RS:0;370bc2ade342:35353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,35353,1731290508685 2024-11-11T02:01:49,793 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,35353,1731290508685' 2024-11-11T02:01:49,793 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:01:49,793 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:01:49,794 DEBUG [RS:0;370bc2ade342:35353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:01:49,794 INFO [RS:0;370bc2ade342:35353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:01:49,794 INFO [RS:0;370bc2ade342:35353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:01:49,864 WARN [370bc2ade342:35677 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T02:01:49,903 INFO [RS:0;370bc2ade342:35353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:01:49,909 INFO [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C35353%2C1731290508685, suffix=, logDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685, archiveDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs, maxLogs=32 2024-11-11T02:01:49,931 DEBUG [RS:0;370bc2ade342:35353 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290509912, exclude list is [], retry=0 2024-11-11T02:01:49,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:01:49,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:01:49,939 INFO [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290509912 2024-11-11T02:01:49,939 DEBUG [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:01:50,120 DEBUG [370bc2ade342:35677 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:01:50,134 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,35353,1731290508685 2024-11-11T02:01:50,140 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,35353,1731290508685, state=OPENING 2024-11-11T02:01:50,144 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:01:50,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:50,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:01:50,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:01:50,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:01:50,147 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:01:50,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,35353,1731290508685}] 2024-11-11T02:01:50,327 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:01:50,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58739, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:01:50,338 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:01:50,339 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:01:50,339 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:01:50,343 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C35353%2C1731290508685.meta, suffix=.meta, logDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685, archiveDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs, maxLogs=32 2024-11-11T02:01:50,359 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.meta.1731290510346.meta, exclude list is [], retry=0 2024-11-11T02:01:50,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:01:50,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:01:50,365 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.meta.1731290510346.meta 2024-11-11T02:01:50,366 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:01:50,366 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:01:50,369 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:01:50,372 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:01:50,377 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:01:50,381 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:01:50,382 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:50,382 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:01:50,382 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:01:50,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:01:50,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:01:50,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:50,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:50,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:01:50,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:01:50,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:50,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:50,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:01:50,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:01:50,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:50,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:50,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:01:50,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:01:50,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:50,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:01:50,395 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:01:50,397 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740 2024-11-11T02:01:50,400 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740 2024-11-11T02:01:50,403 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:01:50,403 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:01:50,404 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:01:50,408 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:01:50,410 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821741, jitterRate=0.044898390769958496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:01:50,410 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:01:50,413 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731290510383Writing region info on filesystem at 1731290510383Initializing all the Stores at 1731290510384 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290510385 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290510385Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290510385Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290510385Cleaning up temporary data from old regions at 1731290510403 (+18 ms)Running coprocessor post-open hooks at 1731290510410 (+7 ms)Region opened successfully at 1731290510412 (+2 ms) 2024-11-11T02:01:50,420 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731290510316 2024-11-11T02:01:50,432 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:01:50,433 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:01:50,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,35353,1731290508685 2024-11-11T02:01:50,436 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,35353,1731290508685, state=OPEN 2024-11-11T02:01:50,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:01:50,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:01:50,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:01:50,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:01:50,440 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,35353,1731290508685 2024-11-11T02:01:50,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:01:50,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,35353,1731290508685 in 292 msec 2024-11-11T02:01:50,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:01:50,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 750 msec 2024-11-11T02:01:50,457 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:01:50,457 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:01:50,475 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:01:50,476 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,35353,1731290508685, seqNum=-1] 2024-11-11T02:01:50,499 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:01:50,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33549, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:01:50,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0020 sec 2024-11-11T02:01:50,520 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731290510520, completionTime=-1 2024-11-11T02:01:50,522 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:01:50,523 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:01:50,547 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:01:50,548 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731290570548 2024-11-11T02:01:50,548 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731290630548 2024-11-11T02:01:50,548 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-11T02:01:50,550 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,550 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,550 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,552 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:35677, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,552 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,553 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,559 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:01:50,579 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.770sec 2024-11-11T02:01:50,580 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:01:50,581 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:01:50,582 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:01:50,583 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:01:50,583 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:01:50,584 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:01:50,585 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:01:50,593 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:01:50,594 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:01:50,594 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,35677,1731290508127-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:01:50,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cf4e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:01:50,678 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T02:01:50,678 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T02:01:50,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,35677,-1 for getting cluster id 2024-11-11T02:01:50,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:01:50,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59902562-b714-4f7a-b65d-ec9f146b03a9' 2024-11-11T02:01:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:01:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59902562-b714-4f7a-b65d-ec9f146b03a9" 2024-11-11T02:01:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9db649e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:01:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,35677,-1] 2024-11-11T02:01:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:01:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:01:50,702 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:01:50,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffbf561, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:01:50,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:01:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,35353,1731290508685, seqNum=-1] 2024-11-11T02:01:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:01:50,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:01:50,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,35677,1731290508127 2024-11-11T02:01:50,736 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:01:50,744 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-11T02:01:50,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:01:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,35677,1731290508127 2024-11-11T02:01:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fa3fa55 2024-11-11T02:01:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:01:50,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:01:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-11T02:01:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-11T02:01:50,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:01:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-11T02:01:50,789 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:01:50,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-11T02:01:50,792 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:50,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:01:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:01:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741835_1011 (size=389) 2024-11-11T02:01:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741835_1011 (size=389) 2024-11-11T02:01:50,818 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0759212a41ff3a565442ac65f3aa9a2e, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571 2024-11-11T02:01:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741836_1012 (size=72) 2024-11-11T02:01:50,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741836_1012 (size=72) 2024-11-11T02:01:50,833 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:50,834 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 0759212a41ff3a565442ac65f3aa9a2e, disabling compactions & flushes 2024-11-11T02:01:50,834 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:50,834 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:50,834 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. after waiting 0 ms 2024-11-11T02:01:50,834 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:50,834 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:50,834 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0759212a41ff3a565442ac65f3aa9a2e: Waiting for close lock at 1731290510834Disabling compacts and flushes for region at 1731290510834Disabling writes for close at 1731290510834Writing region close event to WAL at 1731290510834Closed at 1731290510834 2024-11-11T02:01:50,836 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:01:50,840 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731290510836"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290510836"}]},"ts":"1731290510836"} 2024-11-11T02:01:50,845 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:01:50,847 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:01:50,849 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290510847"}]},"ts":"1731290510847"} 2024-11-11T02:01:50,854 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-11T02:01:50,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e, ASSIGN}] 2024-11-11T02:01:50,858 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e, ASSIGN 2024-11-11T02:01:50,860 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e, ASSIGN; state=OFFLINE, location=370bc2ade342,35353,1731290508685; forceNewPlan=false, retain=false 2024-11-11T02:01:51,012 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0759212a41ff3a565442ac65f3aa9a2e, regionState=OPENING, regionLocation=370bc2ade342,35353,1731290508685 2024-11-11T02:01:51,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e, ASSIGN because future has completed 2024-11-11T02:01:51,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0759212a41ff3a565442ac65f3aa9a2e, server=370bc2ade342,35353,1731290508685}] 2024-11-11T02:01:51,185 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:51,186 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0759212a41ff3a565442ac65f3aa9a2e, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:01:51,186 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,186 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:01:51,187 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,187 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,189 INFO [StoreOpener-0759212a41ff3a565442ac65f3aa9a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,191 INFO [StoreOpener-0759212a41ff3a565442ac65f3aa9a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0759212a41ff3a565442ac65f3aa9a2e columnFamilyName info 2024-11-11T02:01:51,191 DEBUG [StoreOpener-0759212a41ff3a565442ac65f3aa9a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:01:51,192 INFO [StoreOpener-0759212a41ff3a565442ac65f3aa9a2e-1 {}] regionserver.HStore(327): Store=0759212a41ff3a565442ac65f3aa9a2e/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:01:51,192 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,194 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,194 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,195 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,195 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,198 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,202 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:01:51,203 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0759212a41ff3a565442ac65f3aa9a2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811475, jitterRate=0.031843945384025574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:01:51,203 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:01:51,204 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0759212a41ff3a565442ac65f3aa9a2e: Running coprocessor pre-open hook at 1731290511187Writing region info on filesystem at 1731290511187Initializing all the Stores at 1731290511188 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290511188Cleaning up temporary data from old regions at 1731290511195 (+7 ms)Running coprocessor post-open hooks at 1731290511203 (+8 ms)Region opened successfully at 1731290511204 (+1 ms) 2024-11-11T02:01:51,206 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e., pid=6, masterSystemTime=1731290511173 2024-11-11T02:01:51,209 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:51,209 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:01:51,210 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0759212a41ff3a565442ac65f3aa9a2e, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,35353,1731290508685 2024-11-11T02:01:51,213 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35677 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=370bc2ade342,35353,1731290508685, table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T02:01:51,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0759212a41ff3a565442ac65f3aa9a2e, server=370bc2ade342,35353,1731290508685 because future has completed 2024-11-11T02:01:51,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T02:01:51,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0759212a41ff3a565442ac65f3aa9a2e, server=370bc2ade342,35353,1731290508685 in 198 msec 2024-11-11T02:01:51,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T02:01:51,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0759212a41ff3a565442ac65f3aa9a2e, ASSIGN in 365 msec 2024-11-11T02:01:51,227 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:01:51,227 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290511227"}]},"ts":"1731290511227"} 2024-11-11T02:01:51,230 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-11T02:01:51,231 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:01:51,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 464 msec 2024-11-11T02:01:55,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T02:01:55,937 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:01:55,939 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-11T02:01:58,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:01:58,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T02:01:58,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-11T02:01:58,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-11T02:01:58,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:01:58,465 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T02:01:58,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T02:01:58,465 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T02:02:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35677 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:02:00,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-11T02:02:00,885 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-11T02:02:00,891 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-11T02:02:00,891 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:00,908 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290520892, exclude list is [], retry=0 2024-11-11T02:02:00,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:02:00,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:02:00,915 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290509912 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290520892 2024-11-11T02:02:00,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:02:00,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290509912 is not closed yet, will try archiving it next time 2024-11-11T02:02:00,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741833_1009 (size=451) 2024-11-11T02:02:00,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741833_1009 (size=451) 2024-11-11T02:02:00,921 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290509912 to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs/370bc2ade342%2C35353%2C1731290508685.1731290509912 2024-11-11T02:02:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e., hostname=370bc2ade342,35353,1731290508685, seqNum=2] 2024-11-11T02:02:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] regionserver.HRegion(8855): Flush requested on 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:02:14,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0759212a41ff3a565442ac65f3aa9a2e 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:02:15,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/333c5f7192ea4bd4a918312385b7ade4 is 1055, key is row0001/info:/1731290520929/Put/seqid=0 2024-11-11T02:02:15,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741838_1014 (size=13386) 2024-11-11T02:02:15,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741838_1014 (size=13386) 2024-11-11T02:02:15,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/333c5f7192ea4bd4a918312385b7ade4 2024-11-11T02:02:15,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/333c5f7192ea4bd4a918312385b7ade4 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/333c5f7192ea4bd4a918312385b7ade4 2024-11-11T02:02:15,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/333c5f7192ea4bd4a918312385b7ade4, entries=8, sequenceid=12, filesize=13.1 K 2024-11-11T02:02:15,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=0 B/0 for 0759212a41ff3a565442ac65f3aa9a2e in 127ms, sequenceid=12, compaction requested=false 2024-11-11T02:02:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0759212a41ff3a565442ac65f3aa9a2e: 2024-11-11T02:02:17,254 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:02:21,026 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290541008, exclude list is [], retry=0 2024-11-11T02:02:21,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:02:21,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:02:21,033 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290520892 with entries=12, filesize=11.86 KB; new WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290541008 2024-11-11T02:02:21,033 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:02:21,033 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290520892 is not closed yet, will try archiving it next time 2024-11-11T02:02:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741837_1013 (size=12149) 2024-11-11T02:02:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741837_1013 (size=12149) 2024-11-11T02:02:26,039 INFO [AsyncFSWAL-0-hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571-prefix:370bc2ade342,35353,1731290508685 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5003 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK], DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK]] 2024-11-11T02:02:26,043 WARN [AsyncFSWAL-0-hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571-prefix:370bc2ade342,35353,1731290508685 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5003 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK], DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK]] 2024-11-11T02:02:26,045 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 370bc2ade342%2C35353%2C1731290508685:(num 1731290541008) roll requested 2024-11-11T02:02:26,064 DEBUG [regionserver/370bc2ade342:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290546046, exclude list is [], retry=0 2024-11-11T02:02:26,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:02:26,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:02:26,070 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290541008 with entries=1, filesize=1.20 KB; new WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290546046 2024-11-11T02:02:26,070 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35049:35049),(127.0.0.1/127.0.0.1:39855:39855)] 2024-11-11T02:02:26,070 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290541008 is not closed yet, will try archiving it next time 2024-11-11T02:02:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741839_1015 (size=1233) 2024-11-11T02:02:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741839_1015 (size=1233) 2024-11-11T02:02:28,043 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T02:02:28,061 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290548043, exclude list is [], retry=0 2024-11-11T02:02:28,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39139,DS-b7e9aabc-904e-41d3-bee4-06d66c95067d,DISK] 2024-11-11T02:02:28,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33853,DS-7ecb21e1-98c7-46e7-8c7b-7ba4b735be3c,DISK] 2024-11-11T02:02:28,066 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290546046 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290548043 2024-11-11T02:02:28,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39855:39855),(127.0.0.1/127.0.0.1:35049:35049)] 2024-11-11T02:02:28,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290546046 is not closed yet, will try archiving it next time 2024-11-11T02:02:28,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741840_1016 (size=93) 2024-11-11T02:02:28,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741840_1016 (size=93) 2024-11-11T02:02:28,070 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290546046 to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs/370bc2ade342%2C35353%2C1731290508685.1731290546046 2024-11-11T02:02:30,607 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T02:02:30,607 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T02:02:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] regionserver.HRegion(8855): Flush requested on 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:02:36,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0759212a41ff3a565442ac65f3aa9a2e 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:02:36,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/fa6f59b1aa114333b2f9df720eeb9d0f is 1055, key is row0009/info:/1731290536995/Put/seqid=0 2024-11-11T02:02:36,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741842_1018 (size=13386) 2024-11-11T02:02:36,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741842_1018 (size=13386) 2024-11-11T02:02:36,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/fa6f59b1aa114333b2f9df720eeb9d0f 2024-11-11T02:02:36,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/fa6f59b1aa114333b2f9df720eeb9d0f as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/fa6f59b1aa114333b2f9df720eeb9d0f 2024-11-11T02:02:36,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/fa6f59b1aa114333b2f9df720eeb9d0f, entries=8, sequenceid=23, filesize=13.1 K 2024-11-11T02:02:36,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=0 B/0 for 0759212a41ff3a565442ac65f3aa9a2e in 41ms, sequenceid=23, compaction requested=false 2024-11-11T02:02:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0759212a41ff3a565442ac65f3aa9a2e: 2024-11-11T02:02:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.1 K, sizeToCheck=16.0 K 2024-11-11T02:02:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:02:36,144 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/333c5f7192ea4bd4a918312385b7ade4 because midkey is the same as first or last row 2024-11-11T02:02:36,187 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0759212a41ff3a565442ac65f3aa9a2e, had cached 0 bytes from a total of 26772 2024-11-11T02:02:47,255 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:02:48,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:02:48,135 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:02:48,136 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:02:48,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:48,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:48,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:02:48,146 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:02:48,146 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=983861407, stopped=false 2024-11-11T02:02:48,146 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,35677,1731290508127 2024-11-11T02:02:48,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:02:48,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:02:48,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:48,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:02:48,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:48,148 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:02:48,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:02:48,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:48,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:02:48,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:02:48,148 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,35353,1731290508685' ***** 2024-11-11T02:02:48,148 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:02:48,149 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:02:48,149 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:02:48,149 INFO [RS:0;370bc2ade342:35353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:02:48,149 INFO [RS:0;370bc2ade342:35353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:02:48,149 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(3091): Received CLOSE for 0759212a41ff3a565442ac65f3aa9a2e 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,35353,1731290508685 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:35353. 2024-11-11T02:02:48,150 DEBUG [RS:0;370bc2ade342:35353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:02:48,150 DEBUG [RS:0;370bc2ade342:35353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:48,150 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0759212a41ff3a565442ac65f3aa9a2e, disabling compactions & flushes 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:02:48,150 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:02:48,150 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:48,150 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:02:48,150 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. after waiting 0 ms 2024-11-11T02:02:48,150 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:48,151 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0759212a41ff3a565442ac65f3aa9a2e 1/1 column families, dataSize=5.13 KB heapSize=5.76 KB 2024-11-11T02:02:48,151 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T02:02:48,151 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1325): Online Regions={0759212a41ff3a565442ac65f3aa9a2e=TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e., 1588230740=hbase:meta,,1.1588230740} 2024-11-11T02:02:48,151 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:02:48,151 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:02:48,151 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:02:48,151 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:02:48,151 DEBUG [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1351): Waiting on 0759212a41ff3a565442ac65f3aa9a2e, 1588230740 2024-11-11T02:02:48,151 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:02:48,151 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-11T02:02:48,161 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/ae5ef35a70b84f55afd55ba4008ada07 is 1055, key is row0017/info:/1731290558105/Put/seqid=0 2024-11-11T02:02:48,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741843_1019 (size=10222) 2024-11-11T02:02:48,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741843_1019 (size=10222) 2024-11-11T02:02:48,172 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/ae5ef35a70b84f55afd55ba4008ada07 2024-11-11T02:02:48,175 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/info/04334e99077f47cdaff3c908b5a55583 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e./info:regioninfo/1731290511210/Put/seqid=0 2024-11-11T02:02:48,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741844_1020 (size=7016) 2024-11-11T02:02:48,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741844_1020 (size=7016) 2024-11-11T02:02:48,182 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/info/04334e99077f47cdaff3c908b5a55583 2024-11-11T02:02:48,184 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/.tmp/info/ae5ef35a70b84f55afd55ba4008ada07 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/ae5ef35a70b84f55afd55ba4008ada07 2024-11-11T02:02:48,195 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/info/ae5ef35a70b84f55afd55ba4008ada07, entries=5, sequenceid=31, filesize=10.0 K 2024-11-11T02:02:48,197 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.13 KB/5255, heapSize ~5.74 KB/5880, currentSize=0 B/0 for 0759212a41ff3a565442ac65f3aa9a2e in 46ms, sequenceid=31, compaction requested=true 2024-11-11T02:02:48,203 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/default/TestLogRolling-testSlowSyncLogRolling/0759212a41ff3a565442ac65f3aa9a2e/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=1 2024-11-11T02:02:48,204 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:48,205 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0759212a41ff3a565442ac65f3aa9a2e: Waiting for close lock at 1731290568150Running coprocessor pre-close hooks at 1731290568150Disabling compacts and flushes for region at 1731290568150Disabling writes for close at 1731290568150Obtaining lock to block concurrent updates at 1731290568151 (+1 ms)Preparing flush snapshotting stores in 0759212a41ff3a565442ac65f3aa9a2e at 1731290568151Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e., syncing WAL and waiting on mvcc, flushsize=dataSize=5255, getHeapSize=5880, getOffHeapSize=0, getCellsCount=5 at 1731290568151Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. at 1731290568152 (+1 ms)Flushing 0759212a41ff3a565442ac65f3aa9a2e/info: creating writer at 1731290568152Flushing 0759212a41ff3a565442ac65f3aa9a2e/info: appending metadata at 1731290568160 (+8 ms)Flushing 0759212a41ff3a565442ac65f3aa9a2e/info: closing flushed file at 1731290568160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ca14a43: reopening flushed file at 1731290568183 (+23 ms)Finished flush of dataSize ~5.13 KB/5255, heapSize ~5.74 KB/5880, currentSize=0 B/0 for 0759212a41ff3a565442ac65f3aa9a2e in 46ms, sequenceid=31, compaction requested=true at 1731290568197 (+14 ms)Writing region close event to WAL at 1731290568198 (+1 ms)Running coprocessor post-close hooks at 1731290568204 (+6 ms)Closed at 1731290568204 2024-11-11T02:02:48,205 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731290510760.0759212a41ff3a565442ac65f3aa9a2e. 2024-11-11T02:02:48,208 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/ns/1e79b037410a45c6a5da97c566ecf2dc is 43, key is default/ns:d/1731290510506/Put/seqid=0 2024-11-11T02:02:48,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741845_1021 (size=5153) 2024-11-11T02:02:48,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741845_1021 (size=5153) 2024-11-11T02:02:48,215 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/ns/1e79b037410a45c6a5da97c566ecf2dc 2024-11-11T02:02:48,238 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/table/ccbb301293424d3e9624bc8155846336 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731290511227/Put/seqid=0 2024-11-11T02:02:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741846_1022 (size=5396) 2024-11-11T02:02:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741846_1022 (size=5396) 2024-11-11T02:02:48,246 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/table/ccbb301293424d3e9624bc8155846336 2024-11-11T02:02:48,256 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/info/04334e99077f47cdaff3c908b5a55583 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/info/04334e99077f47cdaff3c908b5a55583 2024-11-11T02:02:48,265 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/info/04334e99077f47cdaff3c908b5a55583, entries=10, sequenceid=11, filesize=6.9 K 2024-11-11T02:02:48,267 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/ns/1e79b037410a45c6a5da97c566ecf2dc as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/ns/1e79b037410a45c6a5da97c566ecf2dc 2024-11-11T02:02:48,277 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/ns/1e79b037410a45c6a5da97c566ecf2dc, entries=2, sequenceid=11, filesize=5.0 K 2024-11-11T02:02:48,279 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/.tmp/table/ccbb301293424d3e9624bc8155846336 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/table/ccbb301293424d3e9624bc8155846336 2024-11-11T02:02:48,288 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/table/ccbb301293424d3e9624bc8155846336, entries=2, sequenceid=11, filesize=5.3 K 2024-11-11T02:02:48,289 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 138ms, sequenceid=11, compaction requested=false 2024-11-11T02:02:48,296 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-11T02:02:48,297 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:02:48,297 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:02:48,298 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290568151Running coprocessor pre-close hooks at 1731290568151Disabling compacts and flushes for region at 1731290568151Disabling writes for close at 1731290568151Obtaining lock to block concurrent updates at 1731290568151Preparing flush snapshotting stores in 1588230740 at 1731290568151Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731290568152 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731290568152Flushing 1588230740/info: creating writer at 1731290568152Flushing 1588230740/info: appending metadata at 1731290568174 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731290568174Flushing 1588230740/ns: creating writer at 1731290568191 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731290568208 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731290568208Flushing 1588230740/table: creating writer at 1731290568223 (+15 ms)Flushing 1588230740/table: appending metadata at 1731290568237 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731290568237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f1fb4e: reopening flushed file at 1731290568254 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b5a3684: reopening flushed file at 1731290568265 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b88aa43: reopening flushed file at 1731290568277 (+12 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 138ms, sequenceid=11, compaction requested=false at 1731290568290 (+13 ms)Writing region close event to WAL at 1731290568291 (+1 ms)Running coprocessor post-close hooks at 1731290568297 (+6 ms)Closed at 1731290568297 2024-11-11T02:02:48,298 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:02:48,351 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,35353,1731290508685; all regions closed. 2024-11-11T02:02:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741834_1010 (size=3066) 2024-11-11T02:02:48,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741834_1010 (size=3066) 2024-11-11T02:02:48,365 DEBUG [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs 2024-11-11T02:02:48,365 INFO [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C35353%2C1731290508685.meta:.meta(num 1731290510346) 2024-11-11T02:02:48,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741841_1017 (size=13164) 2024-11-11T02:02:48,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741841_1017 (size=13164) 2024-11-11T02:02:48,368 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/WALs/370bc2ade342,35353,1731290508685/370bc2ade342%2C35353%2C1731290508685.1731290548043 not finished, retry = 0 2024-11-11T02:02:48,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:02:48,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:02:48,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-11T02:02:48,475 DEBUG [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/oldWALs 2024-11-11T02:02:48,475 INFO [RS:0;370bc2ade342:35353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C35353%2C1731290508685:(num 1731290548043) 2024-11-11T02:02:48,475 DEBUG [RS:0;370bc2ade342:35353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:48,475 INFO [RS:0;370bc2ade342:35353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:02:48,475 INFO [RS:0;370bc2ade342:35353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:02:48,476 INFO [RS:0;370bc2ade342:35353 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T02:02:48,476 INFO [RS:0;370bc2ade342:35353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:02:48,476 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:02:48,476 INFO [RS:0;370bc2ade342:35353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35353 2024-11-11T02:02:48,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,35353,1731290508685 2024-11-11T02:02:48,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:02:48,480 INFO [RS:0;370bc2ade342:35353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:02:48,481 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,35353,1731290508685] 2024-11-11T02:02:48,482 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,35353,1731290508685 already deleted, retry=false 2024-11-11T02:02:48,482 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,35353,1731290508685 expired; onlineServers=0 2024-11-11T02:02:48,483 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,35677,1731290508127' ***** 2024-11-11T02:02:48,483 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:02:48,483 INFO [M:0;370bc2ade342:35677 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:02:48,483 INFO [M:0;370bc2ade342:35677 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:02:48,483 DEBUG [M:0;370bc2ade342:35677 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:02:48,483 DEBUG [M:0;370bc2ade342:35677 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:02:48,483 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:02:48,483 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290509615 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290509615,5,FailOnTimeoutGroup] 2024-11-11T02:02:48,483 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290509614 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290509614,5,FailOnTimeoutGroup] 2024-11-11T02:02:48,483 INFO [M:0;370bc2ade342:35677 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:02:48,484 INFO [M:0;370bc2ade342:35677 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:02:48,484 DEBUG [M:0;370bc2ade342:35677 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:02:48,484 INFO [M:0;370bc2ade342:35677 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:02:48,484 INFO [M:0;370bc2ade342:35677 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:02:48,484 INFO [M:0;370bc2ade342:35677 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:02:48,485 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:02:48,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:02:48,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:48,485 DEBUG [M:0;370bc2ade342:35677 {}] zookeeper.ZKUtil(347): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:02:48,485 WARN [M:0;370bc2ade342:35677 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:02:48,486 INFO [M:0;370bc2ade342:35677 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/.lastflushedseqids 2024-11-11T02:02:48,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741847_1023 (size=130) 2024-11-11T02:02:48,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741847_1023 (size=130) 2024-11-11T02:02:48,499 INFO [M:0;370bc2ade342:35677 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:02:48,499 INFO [M:0;370bc2ade342:35677 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:02:48,499 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:02:48,499 INFO [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:48,499 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:48,499 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:02:48,499 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:48,500 INFO [M:0;370bc2ade342:35677 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-11-11T02:02:48,515 DEBUG [M:0;370bc2ade342:35677 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bfe1108f82f49c7acb887c46f1e472a is 82, key is hbase:meta,,1/info:regioninfo/1731290510434/Put/seqid=0 2024-11-11T02:02:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741848_1024 (size=5672) 2024-11-11T02:02:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741848_1024 (size=5672) 2024-11-11T02:02:48,521 INFO [M:0;370bc2ade342:35677 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bfe1108f82f49c7acb887c46f1e472a 2024-11-11T02:02:48,545 DEBUG [M:0;370bc2ade342:35677 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed606bbf4541430c87c38e33e601ecf2 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731290511233/Put/seqid=0 2024-11-11T02:02:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741849_1025 (size=6247) 2024-11-11T02:02:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741849_1025 (size=6247) 2024-11-11T02:02:48,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:02:48,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35353-0x1002c84049b0001, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:02:48,583 INFO [RS:0;370bc2ade342:35353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:02:48,583 INFO [RS:0;370bc2ade342:35353 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,35353,1731290508685; zookeeper connection closed. 2024-11-11T02:02:48,584 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e1c4221 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e1c4221 2024-11-11T02:02:48,584 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:02:48,957 INFO [M:0;370bc2ade342:35677 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed606bbf4541430c87c38e33e601ecf2 2024-11-11T02:02:48,969 INFO [M:0;370bc2ade342:35677 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed606bbf4541430c87c38e33e601ecf2 2024-11-11T02:02:48,984 DEBUG [M:0;370bc2ade342:35677 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fa6a634785743bc882e24934d52e0e6 is 69, key is 370bc2ade342,35353,1731290508685/rs:state/1731290509659/Put/seqid=0 2024-11-11T02:02:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741850_1026 (size=5156) 2024-11-11T02:02:48,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741850_1026 (size=5156) 2024-11-11T02:02:48,990 INFO [M:0;370bc2ade342:35677 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fa6a634785743bc882e24934d52e0e6 2024-11-11T02:02:49,012 DEBUG [M:0;370bc2ade342:35677 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf7c5d7d32f74b688a8714290ea43fe6 is 52, key is load_balancer_on/state:d/1731290510740/Put/seqid=0 2024-11-11T02:02:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741851_1027 (size=5056) 2024-11-11T02:02:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741851_1027 (size=5056) 2024-11-11T02:02:49,018 INFO [M:0;370bc2ade342:35677 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf7c5d7d32f74b688a8714290ea43fe6 2024-11-11T02:02:49,026 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bfe1108f82f49c7acb887c46f1e472a as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bfe1108f82f49c7acb887c46f1e472a 2024-11-11T02:02:49,034 INFO [M:0;370bc2ade342:35677 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bfe1108f82f49c7acb887c46f1e472a, entries=8, sequenceid=59, filesize=5.5 K 2024-11-11T02:02:49,035 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed606bbf4541430c87c38e33e601ecf2 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed606bbf4541430c87c38e33e601ecf2 2024-11-11T02:02:49,042 INFO [M:0;370bc2ade342:35677 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed606bbf4541430c87c38e33e601ecf2 2024-11-11T02:02:49,042 INFO [M:0;370bc2ade342:35677 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed606bbf4541430c87c38e33e601ecf2, entries=6, sequenceid=59, filesize=6.1 K 2024-11-11T02:02:49,043 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fa6a634785743bc882e24934d52e0e6 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6fa6a634785743bc882e24934d52e0e6 2024-11-11T02:02:49,050 INFO [M:0;370bc2ade342:35677 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6fa6a634785743bc882e24934d52e0e6, entries=1, sequenceid=59, filesize=5.0 K 2024-11-11T02:02:49,051 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf7c5d7d32f74b688a8714290ea43fe6 as hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf7c5d7d32f74b688a8714290ea43fe6 2024-11-11T02:02:49,058 INFO [M:0;370bc2ade342:35677 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf7c5d7d32f74b688a8714290ea43fe6, entries=1, sequenceid=59, filesize=4.9 K 2024-11-11T02:02:49,059 INFO [M:0;370bc2ade342:35677 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 560ms, sequenceid=59, compaction requested=false 2024-11-11T02:02:49,061 INFO [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:49,061 DEBUG [M:0;370bc2ade342:35677 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290568499Disabling compacts and flushes for region at 1731290568499Disabling writes for close at 1731290568499Obtaining lock to block concurrent updates at 1731290568500 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731290568500Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731290568500Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731290568501 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731290568501Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731290568515 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731290568515Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731290568529 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731290568545 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731290568545Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731290568970 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731290568983 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731290568983Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731290568997 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731290569011 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731290569011Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75379ea7: reopening flushed file at 1731290569025 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3808f69a: reopening flushed file at 1731290569034 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7268d355: reopening flushed file at 1731290569042 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40d67884: reopening flushed file at 1731290569050 (+8 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 560ms, sequenceid=59, compaction requested=false at 1731290569059 (+9 ms)Writing region close event to WAL at 1731290569061 (+2 ms)Closed at 1731290569061 2024-11-11T02:02:49,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741830_1006 (size=27973) 2024-11-11T02:02:49,064 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/MasterData/WALs/370bc2ade342,35677,1731290508127/370bc2ade342%2C35677%2C1731290508127.1731290509091 not finished, retry = 0 2024-11-11T02:02:49,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33853 is added to blk_1073741830_1006 (size=27973) 2024-11-11T02:02:49,166 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:02:49,166 INFO [M:0;370bc2ade342:35677 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:02:49,166 INFO [M:0;370bc2ade342:35677 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35677 2024-11-11T02:02:49,167 INFO [M:0;370bc2ade342:35677 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:02:49,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:02:49,270 INFO [M:0;370bc2ade342:35677 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:02:49,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35677-0x1002c84049b0000, quorum=127.0.0.1:57806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:02:49,279 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b9fe80c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:49,282 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@431c6310{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:02:49,282 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:02:49,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@775a6b37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:02:49,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d434c3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,STOPPED} 2024-11-11T02:02:49,285 WARN [BP-1855330386-172.17.0.2-1731290505625 heartbeating to localhost/127.0.0.1:35985 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:02:49,285 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:02:49,285 WARN [BP-1855330386-172.17.0.2-1731290505625 heartbeating to localhost/127.0.0.1:35985 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1855330386-172.17.0.2-1731290505625 (Datanode Uuid 5521b6ce-b6a0-472f-b601-0cc73c253eca) service to localhost/127.0.0.1:35985 2024-11-11T02:02:49,285 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:02:49,286 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data3/current/BP-1855330386-172.17.0.2-1731290505625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:02:49,286 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data4/current/BP-1855330386-172.17.0.2-1731290505625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:02:49,286 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:02:49,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@790fbcfe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:49,289 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e3c8436{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:02:49,289 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:02:49,289 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49b96e54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:02:49,289 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2973dc0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,STOPPED} 2024-11-11T02:02:49,290 WARN [BP-1855330386-172.17.0.2-1731290505625 heartbeating to localhost/127.0.0.1:35985 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:02:49,290 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:02:49,291 WARN [BP-1855330386-172.17.0.2-1731290505625 heartbeating to localhost/127.0.0.1:35985 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1855330386-172.17.0.2-1731290505625 (Datanode Uuid 80ff741e-d986-4785-bc90-082a3f73dfea) service to localhost/127.0.0.1:35985 2024-11-11T02:02:49,291 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:02:49,291 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data1/current/BP-1855330386-172.17.0.2-1731290505625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:02:49,291 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/cluster_fd2a5224-c331-d707-e46b-ecc486979bfc/data/data2/current/BP-1855330386-172.17.0.2-1731290505625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:02:49,292 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:02:49,300 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@612e4b2b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:02:49,301 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@65270512{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:02:49,301 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:02:49,301 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6932e96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:02:49,302 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20abc9e4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir/,STOPPED} 2024-11-11T02:02:49,310 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:02:49,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T02:02:49,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testSlowSyncLogRolling Thread=65 (was 12) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35985 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/370bc2ade342:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35985 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Slow-sync-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:35985 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:35985 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@2692b93e java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35985 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/370bc2ade342:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/370bc2ade342:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/370bc2ade342:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:35985 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=390 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=41 (was 22) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6525 (was 7073) 2024-11-11T02:02:49,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRollOnDatanodeDeath Thread=66, OpenFileDescriptor=390, MaxFileDescriptor=1048576, SystemLoadAverage=41, ProcessCount=11, AvailableMemoryMB=6524 2024-11-11T02:02:49,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:02:49,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.log.dir so I do NOT create it in target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32 2024-11-11T02:02:49,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b56b1bd-d84d-59b9-b702-ed8336b2a865/hadoop.tmp.dir so I do NOT create it in target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0, deleteOnExit=true 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/test.cache.data in system properties and HBase conf 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:02:49,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:02:49,350 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:02:49,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:02:49,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:02:49,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:02:49,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:02:49,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:02:49,410 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:49,416 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:49,417 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:49,417 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:49,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:02:49,420 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:49,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7648fb82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:49,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fba372a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:49,516 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@375d1d1a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-38003-hadoop-hdfs-3_4_1-tests_jar-_-any-7859653442885525530/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:02:49,517 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@487bdcb9{HTTP/1.1, (http/1.1)}{localhost:38003} 2024-11-11T02:02:49,517 INFO [Time-limited test {}] server.Server(415): Started @65833ms 2024-11-11T02:02:49,576 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:49,579 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:49,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:49,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:49,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:02:49,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a545b17{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:49,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69f035ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:49,676 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6af8ea6c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-44049-hadoop-hdfs-3_4_1-tests_jar-_-any-2411725703637022487/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:49,676 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26d8ef5e{HTTP/1.1, (http/1.1)}{localhost:44049} 2024-11-11T02:02:49,676 INFO [Time-limited test {}] server.Server(415): Started @65992ms 2024-11-11T02:02:49,678 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:02:49,710 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:49,715 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:49,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:49,716 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:49,716 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:02:49,716 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26c15ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:49,716 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5047de0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:49,737 WARN [Thread-370 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data1/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:49,737 WARN [Thread-371 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data2/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:49,746 INFO [regionserver/370bc2ade342:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:02:49,756 WARN [Thread-349 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:02:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbedccfd51275bae9 with lease ID 0x26506e4ffbc32e4c: Processing first storage report for DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40 from datanode DatanodeRegistration(127.0.0.1:44245, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=42869, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbedccfd51275bae9 with lease ID 0x26506e4ffbc32e4c: from storage DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40 node DatanodeRegistration(127.0.0.1:44245, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=42869, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbedccfd51275bae9 with lease ID 0x26506e4ffbc32e4c: Processing first storage report for DS-f197bfa4-a279-4ff7-977b-abd6c4a5ba4a from datanode DatanodeRegistration(127.0.0.1:44245, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=42869, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbedccfd51275bae9 with lease ID 0x26506e4ffbc32e4c: from storage DS-f197bfa4-a279-4ff7-977b-abd6c4a5ba4a node DatanodeRegistration(127.0.0.1:44245, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=42869, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:49,814 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a58fc3f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-43483-hadoop-hdfs-3_4_1-tests_jar-_-any-11459838285150419836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:49,814 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f06258f{HTTP/1.1, (http/1.1)}{localhost:43483} 2024-11-11T02:02:49,814 INFO [Time-limited test {}] server.Server(415): Started @66130ms 2024-11-11T02:02:49,815 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:02:49,874 WARN [Thread-396 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data3/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:49,874 WARN [Thread-397 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data4/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:49,890 WARN [Thread-385 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:02:49,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa21a73bbd6361576 with lease ID 0x26506e4ffbc32e4d: Processing first storage report for DS-82f8e851-81b8-4c90-bba8-a53a362332fb from datanode DatanodeRegistration(127.0.0.1:34583, datanodeUuid=8c86de0f-bd6f-4f7f-9492-2ba399dee161, infoPort=43025, infoSecurePort=0, ipcPort=35045, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:49,892 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa21a73bbd6361576 with lease ID 0x26506e4ffbc32e4d: from storage DS-82f8e851-81b8-4c90-bba8-a53a362332fb node DatanodeRegistration(127.0.0.1:34583, datanodeUuid=8c86de0f-bd6f-4f7f-9492-2ba399dee161, infoPort=43025, infoSecurePort=0, ipcPort=35045, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:49,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa21a73bbd6361576 with lease ID 0x26506e4ffbc32e4d: Processing first storage report for DS-b2f200f3-3408-4f65-a155-80cd390b856e from datanode DatanodeRegistration(127.0.0.1:34583, datanodeUuid=8c86de0f-bd6f-4f7f-9492-2ba399dee161, infoPort=43025, infoSecurePort=0, ipcPort=35045, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:49,892 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa21a73bbd6361576 with lease ID 0x26506e4ffbc32e4d: from storage DS-b2f200f3-3408-4f65-a155-80cd390b856e node DatanodeRegistration(127.0.0.1:34583, datanodeUuid=8c86de0f-bd6f-4f7f-9492-2ba399dee161, infoPort=43025, infoSecurePort=0, ipcPort=35045, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:49,941 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32 2024-11-11T02:02:49,944 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/zookeeper_0, clientPort=63037, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:02:49,945 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63037 2024-11-11T02:02:49,945 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:49,947 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:49,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:02:49,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:02:49,960 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1 with version=8 2024-11-11T02:02:49,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase-staging 2024-11-11T02:02:49,962 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:02:49,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:49,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:49,962 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:02:49,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:49,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:02:49,963 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:02:49,963 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:02:49,963 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40297 2024-11-11T02:02:49,965 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40297 connecting to ZooKeeper ensemble=127.0.0.1:63037 2024-11-11T02:02:49,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402970x0, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:02:49,969 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40297-0x1002c84f9460000 connected 2024-11-11T02:02:49,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:49,981 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:49,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:02:49,984 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1, hbase.cluster.distributed=false 2024-11-11T02:02:49,986 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:02:49,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40297 2024-11-11T02:02:49,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40297 2024-11-11T02:02:49,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40297 2024-11-11T02:02:49,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40297 2024-11-11T02:02:49,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40297 2024-11-11T02:02:50,007 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:02:50,007 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:02:50,008 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33569 2024-11-11T02:02:50,009 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33569 connecting to ZooKeeper ensemble=127.0.0.1:63037 2024-11-11T02:02:50,010 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:50,012 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:50,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335690x0, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:02:50,016 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33569-0x1002c84f9460001 connected 2024-11-11T02:02:50,016 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:02:50,016 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:02:50,020 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:02:50,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:02:50,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:02:50,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33569 2024-11-11T02:02:50,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33569 2024-11-11T02:02:50,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33569 2024-11-11T02:02:50,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33569 2024-11-11T02:02:50,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33569 2024-11-11T02:02:50,039 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:40297 2024-11-11T02:02:50,040 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:02:50,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:02:50,042 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:02:50,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,043 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:02:50,044 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,40297,1731290569962 from backup master directory 2024-11-11T02:02:50,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:02:50,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:02:50,045 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:02:50,045 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,050 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/hbase.id] with ID: afe02bb7-e869-4aa7-bb44-3a03cece849b 2024-11-11T02:02:50,050 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/.tmp/hbase.id 2024-11-11T02:02:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:02:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:02:50,073 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/.tmp/hbase.id]:[hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/hbase.id] 2024-11-11T02:02:50,091 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:50,091 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:02:50,093 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-11T02:02:50,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:02:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:02:50,106 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:02:50,107 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:02:50,107 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:02:50,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:02:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:02:50,117 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store 2024-11-11T02:02:50,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:02:50,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:02:50,127 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:50,127 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:02:50,127 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290570127Disabling compacts and flushes for region at 1731290570127Disabling writes for close at 1731290570127Writing region close event to WAL at 1731290570127Closed at 1731290570127 2024-11-11T02:02:50,128 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/.initializing 2024-11-11T02:02:50,129 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,130 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:02:50,133 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C40297%2C1731290569962, suffix=, logDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962, archiveDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/oldWALs, maxLogs=10 2024-11-11T02:02:50,146 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133, exclude list is [], retry=0 2024-11-11T02:02:50,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44245,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] 2024-11-11T02:02:50,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34583,DS-82f8e851-81b8-4c90-bba8-a53a362332fb,DISK] 2024-11-11T02:02:50,152 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 2024-11-11T02:02:50,152 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42869:42869),(127.0.0.1/127.0.0.1:43025:43025)] 2024-11-11T02:02:50,152 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:02:50,153 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:50,153 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,153 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:02:50,156 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:02:50,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:02:50,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:02:50,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:02:50,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:02:50,165 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:02:50,166 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,167 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,168 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,170 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,170 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,171 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:02:50,173 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:02:50,176 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:02:50,177 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732104, jitterRate=-0.06908248364925385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:02:50,179 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731290570153Initializing all the Stores at 1731290570154 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570154Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290570154Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290570154Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290570154Cleaning up temporary data from old regions at 1731290570170 (+16 ms)Region opened successfully at 1731290570179 (+9 ms) 2024-11-11T02:02:50,179 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:02:50,184 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6c3f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:02:50,185 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:02:50,185 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:02:50,185 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:02:50,185 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:02:50,186 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T02:02:50,186 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T02:02:50,187 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:02:50,189 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:02:50,190 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:02:50,191 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:02:50,191 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:02:50,192 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:02:50,193 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:02:50,193 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:02:50,194 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:02:50,195 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:02:50,196 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:02:50,197 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:02:50,199 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:02:50,200 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:02:50,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:02:50,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:02:50,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,202 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,40297,1731290569962, sessionid=0x1002c84f9460000, setting cluster-up flag (Was=false) 2024-11-11T02:02:50,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,208 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:02:50,209 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,215 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:02:50,216 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,40297,1731290569962 2024-11-11T02:02:50,217 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:02:50,219 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:02:50,220 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:02:50,220 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:02:50,220 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,40297,1731290569962 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:02:50,221 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:02:50,222 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731290600224 2024-11-11T02:02:50,224 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:02:50,224 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:02:50,224 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:02:50,225 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,225 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:02:50,225 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:02:50,225 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:02:50,226 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:02:50,226 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:02:50,226 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,226 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:02:50,228 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290570226,5,FailOnTimeoutGroup] 2024-11-11T02:02:50,230 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290570228,5,FailOnTimeoutGroup] 2024-11-11T02:02:50,230 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,230 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:02:50,230 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,230 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,232 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(746): ClusterId : afe02bb7-e869-4aa7-bb44-3a03cece849b 2024-11-11T02:02:50,232 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:02:50,234 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:02:50,234 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:02:50,236 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:02:50,236 DEBUG [RS:0;370bc2ade342:33569 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e752541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:02:50,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:02:50,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:02:50,242 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:02:50,243 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1 2024-11-11T02:02:50,249 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:33569 2024-11-11T02:02:50,249 INFO [RS:0;370bc2ade342:33569 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:02:50,250 INFO [RS:0;370bc2ade342:33569 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:02:50,250 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:02:50,251 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,40297,1731290569962 with port=33569, startcode=1731290570006 2024-11-11T02:02:50,251 DEBUG [RS:0;370bc2ade342:33569 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:02:50,253 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36227, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:02:50,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40297 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:02:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:02:50,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40297 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:50,257 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1 2024-11-11T02:02:50,257 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42151 2024-11-11T02:02:50,257 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:02:50,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:02:50,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:02:50,259 DEBUG [RS:0;370bc2ade342:33569 {}] zookeeper.ZKUtil(111): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,259 WARN [RS:0;370bc2ade342:33569 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:02:50,259 INFO [RS:0;370bc2ade342:33569 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:02:50,259 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,259 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:02:50,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:02:50,261 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,33569,1731290570006] 2024-11-11T02:02:50,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:02:50,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:02:50,264 INFO [RS:0;370bc2ade342:33569 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:02:50,265 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:02:50,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:02:50,268 INFO [RS:0;370bc2ade342:33569 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:02:50,268 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:02:50,268 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,268 INFO [RS:0;370bc2ade342:33569 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:02:50,268 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,268 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:02:50,269 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:02:50,269 INFO [RS:0;370bc2ade342:33569 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:02:50,270 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:02:50,270 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:02:50,271 DEBUG [RS:0;370bc2ade342:33569 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:02:50,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:02:50,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:02:50,273 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:02:50,274 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:02:50,276 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,276 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,276 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,276 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,276 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,277 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,33569,1731290570006-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:02:50,277 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:02:50,278 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767769, jitterRate=-0.023732483386993408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:02:50,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731290570255Initializing all the Stores at 1731290570256 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570256Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570257 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290570257Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570257Cleaning up temporary data from old regions at 1731290570272 (+15 ms)Region opened successfully at 1731290570278 (+6 ms) 2024-11-11T02:02:50,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:02:50,279 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:02:50,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:02:50,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:02:50,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:02:50,281 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:02:50,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290570279Disabling compacts and flushes for region at 1731290570279Disabling writes for close at 1731290570279Writing region close event to WAL at 1731290570281 (+2 ms)Closed at 1731290570281 2024-11-11T02:02:50,283 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:02:50,283 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:02:50,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:02:50,285 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:02:50,287 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:02:50,299 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:02:50,299 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,33569,1731290570006-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,299 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,299 INFO [RS:0;370bc2ade342:33569 {}] regionserver.Replication(171): 370bc2ade342,33569,1731290570006 started 2024-11-11T02:02:50,314 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,314 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,33569,1731290570006, RpcServer on 370bc2ade342/172.17.0.2:33569, sessionid=0x1002c84f9460001 2024-11-11T02:02:50,315 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:02:50,315 DEBUG [RS:0;370bc2ade342:33569 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,315 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,33569,1731290570006' 2024-11-11T02:02:50,315 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:02:50,315 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:02:50,316 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:02:50,316 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:02:50,316 DEBUG [RS:0;370bc2ade342:33569 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,316 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,33569,1731290570006' 2024-11-11T02:02:50,316 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:02:50,317 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:02:50,317 DEBUG [RS:0;370bc2ade342:33569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:02:50,317 INFO [RS:0;370bc2ade342:33569 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:02:50,317 INFO [RS:0;370bc2ade342:33569 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:02:50,418 INFO [RS:0;370bc2ade342:33569 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:02:50,422 INFO [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C33569%2C1731290570006, suffix=, logDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006, archiveDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs, maxLogs=32 2024-11-11T02:02:50,437 WARN [370bc2ade342:40297 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T02:02:50,439 DEBUG [RS:0;370bc2ade342:33569 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426, exclude list is [], retry=0 2024-11-11T02:02:50,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34583,DS-82f8e851-81b8-4c90-bba8-a53a362332fb,DISK] 2024-11-11T02:02:50,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44245,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] 2024-11-11T02:02:50,445 INFO [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 2024-11-11T02:02:50,446 DEBUG [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42869:42869),(127.0.0.1/127.0.0.1:43025:43025)] 2024-11-11T02:02:50,688 DEBUG [370bc2ade342:40297 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:02:50,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,693 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,33569,1731290570006, state=OPENING 2024-11-11T02:02:50,696 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:02:50,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:02:50,699 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:02:50,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:02:50,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:02:50,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,33569,1731290570006}] 2024-11-11T02:02:50,854 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:02:50,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:02:50,860 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:02:50,860 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:02:50,861 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:02:50,863 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C33569%2C1731290570006.meta, suffix=.meta, logDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006, archiveDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs, maxLogs=32 2024-11-11T02:02:50,879 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta, exclude list is [], retry=0 2024-11-11T02:02:50,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44245,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] 2024-11-11T02:02:50,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34583,DS-82f8e851-81b8-4c90-bba8-a53a362332fb,DISK] 2024-11-11T02:02:50,884 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta 2024-11-11T02:02:50,884 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43025:43025),(127.0.0.1/127.0.0.1:42869:42869)] 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:02:50,885 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:02:50,885 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:02:50,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:02:50,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:02:50,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:02:50,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:02:50,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:02:50,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:02:50,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:02:50,894 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:02:50,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:50,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:02:50,895 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:02:50,896 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740 2024-11-11T02:02:50,898 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740 2024-11-11T02:02:50,899 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:02:50,899 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:02:50,900 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:02:50,901 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:02:50,902 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751361, jitterRate=-0.044595927000045776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:02:50,903 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:02:50,903 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731290570886Writing region info on filesystem at 1731290570886Initializing all the Stores at 1731290570887 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570887Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570887Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290570887Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290570887Cleaning up temporary data from old regions at 1731290570899 (+12 ms)Running coprocessor post-open hooks at 1731290570903 (+4 ms)Region opened successfully at 1731290570903 2024-11-11T02:02:50,905 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731290570854 2024-11-11T02:02:50,907 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:02:50,908 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:02:50,908 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,910 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,33569,1731290570006, state=OPEN 2024-11-11T02:02:50,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:02:50,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:02:50,912 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,33569,1731290570006 2024-11-11T02:02:50,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:02:50,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:02:50,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:02:50,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,33569,1731290570006 in 213 msec 2024-11-11T02:02:50,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:02:50,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 633 msec 2024-11-11T02:02:50,921 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:02:50,921 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:02:50,922 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:02:50,923 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,33569,1731290570006, seqNum=-1] 2024-11-11T02:02:50,923 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:02:50,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:02:50,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 711 msec 2024-11-11T02:02:50,932 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731290570931, completionTime=-1 2024-11-11T02:02:50,932 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:02:50,932 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:02:50,935 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:02:50,935 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731290630935 2024-11-11T02:02:50,935 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731290690935 2024-11-11T02:02:50,935 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-11T02:02:50,935 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,936 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,936 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,936 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:40297, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,936 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,936 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:50,939 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:02:50,942 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.897sec 2024-11-11T02:02:50,942 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:02:50,943 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:02:50,946 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:02:50,946 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:02:50,946 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40297,1731290569962-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:02:51,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa855ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:02:51,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,40297,-1 for getting cluster id 2024-11-11T02:02:51,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:02:51,034 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'afe02bb7-e869-4aa7-bb44-3a03cece849b' 2024-11-11T02:02:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:02:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "afe02bb7-e869-4aa7-bb44-3a03cece849b" 2024-11-11T02:02:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbb9729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:02:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,40297,-1] 2024-11-11T02:02:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:02:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:02:51,038 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:02:51,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e52b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:02:51,040 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:02:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,33569,1731290570006, seqNum=-1] 2024-11-11T02:02:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:02:51,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:02:51,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,40297,1731290569962 2024-11-11T02:02:51,048 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:02:51,053 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-11T02:02:51,068 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:51,072 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:51,073 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:51,073 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:51,073 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:02:51,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6299fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:51,074 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59327d46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:51,168 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59d8e30e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-44497-hadoop-hdfs-3_4_1-tests_jar-_-any-512287903442469531/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:51,168 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1d2cd94{HTTP/1.1, (http/1.1)}{localhost:44497} 2024-11-11T02:02:51,168 INFO [Time-limited test {}] server.Server(415): Started @67484ms 2024-11-11T02:02:51,170 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:02:51,203 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:51,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:51,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:51,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:51,208 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:02:51,208 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f965db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:51,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c1a1532{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:51,232 WARN [Thread-509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data5/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,232 WARN [Thread-510 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data6/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,251 WARN [Thread-489 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:02:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x945121503de1e6eb with lease ID 0x26506e4ffbc32e4e: Processing first storage report for DS-8027192a-9bfb-4d8b-8832-3d1fc3b69bc0 from datanode DatanodeRegistration(127.0.0.1:39945, datanodeUuid=ffa2cb22-3a7a-4e70-a5ac-c66cebd206fd, infoPort=42283, infoSecurePort=0, ipcPort=41595, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x945121503de1e6eb with lease ID 0x26506e4ffbc32e4e: from storage DS-8027192a-9bfb-4d8b-8832-3d1fc3b69bc0 node DatanodeRegistration(127.0.0.1:39945, datanodeUuid=ffa2cb22-3a7a-4e70-a5ac-c66cebd206fd, infoPort=42283, infoSecurePort=0, ipcPort=41595, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x945121503de1e6eb with lease ID 0x26506e4ffbc32e4e: Processing first storage report for DS-6212bcda-7ceb-48a1-92b3-bf83ebcdc7f3 from datanode DatanodeRegistration(127.0.0.1:39945, datanodeUuid=ffa2cb22-3a7a-4e70-a5ac-c66cebd206fd, infoPort=42283, infoSecurePort=0, ipcPort=41595, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x945121503de1e6eb with lease ID 0x26506e4ffbc32e4e: from storage DS-6212bcda-7ceb-48a1-92b3-bf83ebcdc7f3 node DatanodeRegistration(127.0.0.1:39945, datanodeUuid=ffa2cb22-3a7a-4e70-a5ac-c66cebd206fd, infoPort=42283, infoSecurePort=0, ipcPort=41595, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a5edbcb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-45853-hadoop-hdfs-3_4_1-tests_jar-_-any-1711048470442245687/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:51,307 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@177b2eb8{HTTP/1.1, (http/1.1)}{localhost:45853} 2024-11-11T02:02:51,308 INFO [Time-limited test {}] server.Server(415): Started @67623ms 2024-11-11T02:02:51,309 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:02:51,345 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:02:51,349 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:02:51,356 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:02:51,356 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:02:51,356 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:02:51,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63f1466f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:02:51,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b434fa5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:02:51,371 WARN [Thread-544 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data7/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,372 WARN [Thread-545 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data8/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,388 WARN [Thread-524 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:02:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe5273ac4ca2a2203 with lease ID 0x26506e4ffbc32e4f: Processing first storage report for DS-c8d575af-798e-4640-9973-f34b52cafee1 from datanode DatanodeRegistration(127.0.0.1:45297, datanodeUuid=d4317a7e-6ca3-4230-984c-daafe1b742f0, infoPort=44891, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe5273ac4ca2a2203 with lease ID 0x26506e4ffbc32e4f: from storage DS-c8d575af-798e-4640-9973-f34b52cafee1 node DatanodeRegistration(127.0.0.1:45297, datanodeUuid=d4317a7e-6ca3-4230-984c-daafe1b742f0, infoPort=44891, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe5273ac4ca2a2203 with lease ID 0x26506e4ffbc32e4f: Processing first storage report for DS-c24a3682-a993-450c-8944-61a9c050bc00 from datanode DatanodeRegistration(127.0.0.1:45297, datanodeUuid=d4317a7e-6ca3-4230-984c-daafe1b742f0, infoPort=44891, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe5273ac4ca2a2203 with lease ID 0x26506e4ffbc32e4f: from storage DS-c24a3682-a993-450c-8944-61a9c050bc00 node DatanodeRegistration(127.0.0.1:45297, datanodeUuid=d4317a7e-6ca3-4230-984c-daafe1b742f0, infoPort=44891, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,454 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7016180d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-39795-hadoop-hdfs-3_4_1-tests_jar-_-any-11994487887738669381/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:02:51,454 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64bc4e58{HTTP/1.1, (http/1.1)}{localhost:39795} 2024-11-11T02:02:51,454 INFO [Time-limited test {}] server.Server(415): Started @67770ms 2024-11-11T02:02:51,456 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:02:51,512 WARN [Thread-569 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data9/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,512 WARN [Thread-570 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data10/current/BP-1431619133-172.17.0.2-1731290569374/current, will proceed with Du for space computation calculation, 2024-11-11T02:02:51,539 WARN [Thread-559 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:02:51,542 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4518d340e3378cc2 with lease ID 0x26506e4ffbc32e50: Processing first storage report for DS-8489b8a0-cfdb-46d2-9901-464601b0ac6f from datanode DatanodeRegistration(127.0.0.1:44095, datanodeUuid=c12e3151-19d6-4d77-93d4-5d8e123bcf58, infoPort=45211, infoSecurePort=0, ipcPort=36515, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,542 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4518d340e3378cc2 with lease ID 0x26506e4ffbc32e50: from storage DS-8489b8a0-cfdb-46d2-9901-464601b0ac6f node DatanodeRegistration(127.0.0.1:44095, datanodeUuid=c12e3151-19d6-4d77-93d4-5d8e123bcf58, infoPort=45211, infoSecurePort=0, ipcPort=36515, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4518d340e3378cc2 with lease ID 0x26506e4ffbc32e50: Processing first storage report for DS-81183c38-0193-4974-8642-664f18f83576 from datanode DatanodeRegistration(127.0.0.1:44095, datanodeUuid=c12e3151-19d6-4d77-93d4-5d8e123bcf58, infoPort=45211, infoSecurePort=0, ipcPort=36515, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374) 2024-11-11T02:02:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4518d340e3378cc2 with lease ID 0x26506e4ffbc32e50: from storage DS-81183c38-0193-4974-8642-664f18f83576 node DatanodeRegistration(127.0.0.1:44095, datanodeUuid=c12e3151-19d6-4d77-93d4-5d8e123bcf58, infoPort=45211, infoSecurePort=0, ipcPort=36515, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:02:51,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:02:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,40297,1731290569962 2024-11-11T02:02:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2470d209 2024-11-11T02:02:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:02:51,577 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:02:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-11T02:02:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-11T02:02:51,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:02:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-11T02:02:51,583 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:02:51,583 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:51,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-11T02:02:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:02:51,585 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:02:51,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44245 is added to blk_1073741835_1011 (size=393) 2024-11-11T02:02:51,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741835_1011 (size=393) 2024-11-11T02:02:51,595 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7dd8321a8f4ac9f6b1d26f3f16288215, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1 2024-11-11T02:02:51,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741836_1012 (size=76) 2024-11-11T02:02:51,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741836_1012 (size=76) 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 7dd8321a8f4ac9f6b1d26f3f16288215, disabling compactions & flushes 2024-11-11T02:02:51,603 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. after waiting 0 ms 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,603 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,603 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7dd8321a8f4ac9f6b1d26f3f16288215: Waiting for close lock at 1731290571603Disabling compacts and flushes for region at 1731290571603Disabling writes for close at 1731290571603Writing region close event to WAL at 1731290571603Closed at 1731290571603 2024-11-11T02:02:51,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:02:51,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731290571605"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290571605"}]},"ts":"1731290571605"} 2024-11-11T02:02:51,609 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:02:51,611 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:02:51,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290571611"}]},"ts":"1731290571611"} 2024-11-11T02:02:51,614 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-11T02:02:51,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7dd8321a8f4ac9f6b1d26f3f16288215, ASSIGN}] 2024-11-11T02:02:51,616 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7dd8321a8f4ac9f6b1d26f3f16288215, ASSIGN 2024-11-11T02:02:51,618 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7dd8321a8f4ac9f6b1d26f3f16288215, ASSIGN; state=OFFLINE, location=370bc2ade342,33569,1731290570006; forceNewPlan=false, retain=false 2024-11-11T02:02:51,769 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7dd8321a8f4ac9f6b1d26f3f16288215, regionState=OPENING, regionLocation=370bc2ade342,33569,1731290570006 2024-11-11T02:02:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7dd8321a8f4ac9f6b1d26f3f16288215, ASSIGN because future has completed 2024-11-11T02:02:51,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dd8321a8f4ac9f6b1d26f3f16288215, server=370bc2ade342,33569,1731290570006}] 2024-11-11T02:02:51,940 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,940 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7dd8321a8f4ac9f6b1d26f3f16288215, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:02:51,941 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,941 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:02:51,941 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,941 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,943 INFO [StoreOpener-7dd8321a8f4ac9f6b1d26f3f16288215-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,945 INFO [StoreOpener-7dd8321a8f4ac9f6b1d26f3f16288215-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dd8321a8f4ac9f6b1d26f3f16288215 columnFamilyName info 2024-11-11T02:02:51,946 DEBUG [StoreOpener-7dd8321a8f4ac9f6b1d26f3f16288215-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:02:51,946 INFO [StoreOpener-7dd8321a8f4ac9f6b1d26f3f16288215-1 {}] regionserver.HStore(327): Store=7dd8321a8f4ac9f6b1d26f3f16288215/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:02:51,946 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,947 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,948 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,948 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,948 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,951 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,953 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:02:51,954 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7dd8321a8f4ac9f6b1d26f3f16288215; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814331, jitterRate=0.03547607362270355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:02:51,954 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:02:51,955 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7dd8321a8f4ac9f6b1d26f3f16288215: Running coprocessor pre-open hook at 1731290571941Writing region info on filesystem at 1731290571941Initializing all the Stores at 1731290571943 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290571943Cleaning up temporary data from old regions at 1731290571948 (+5 ms)Running coprocessor post-open hooks at 1731290571954 (+6 ms)Region opened successfully at 1731290571955 (+1 ms) 2024-11-11T02:02:51,956 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215., pid=6, masterSystemTime=1731290571933 2024-11-11T02:02:51,959 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,959 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:02:51,960 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7dd8321a8f4ac9f6b1d26f3f16288215, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,33569,1731290570006 2024-11-11T02:02:51,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dd8321a8f4ac9f6b1d26f3f16288215, server=370bc2ade342,33569,1731290570006 because future has completed 2024-11-11T02:02:51,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T02:02:51,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7dd8321a8f4ac9f6b1d26f3f16288215, server=370bc2ade342,33569,1731290570006 in 187 msec 2024-11-11T02:02:51,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T02:02:51,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7dd8321a8f4ac9f6b1d26f3f16288215, ASSIGN in 354 msec 2024-11-11T02:02:51,973 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:02:51,973 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290571973"}]},"ts":"1731290571973"} 2024-11-11T02:02:51,976 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-11T02:02:51,977 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:02:51,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 398 msec 2024-11-11T02:02:53,210 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:02:53,220 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:02:53,734 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:02:53,747 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:02:53,762 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:02:56,264 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:02:56,265 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-11T02:02:58,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:02:58,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T02:02:58,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-11T02:02:58,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-11T02:02:58,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:02:58,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T02:03:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:03:01,683 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-11T02:03:01,684 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestLogRolling-testLogRollOnDatanodeDeath get assigned. Timeout = 60000ms 2024-11-11T02:03:01,686 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T02:03:01,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestLogRolling-testLogRollOnDatanodeDeath assigned to meta. Checking AM states. 2024-11-11T02:03:01,693 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T02:03:01,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestLogRolling-testLogRollOnDatanodeDeath assigned. 2024-11-11T02:03:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215., hostname=370bc2ade342,33569,1731290570006, seqNum=2] 2024-11-11T02:03:01,701 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-11T02:03:01,704 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-11T02:03:01,704 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,706 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-291656986_22 at /127.0.0.1:41284 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41284 dst: /127.0.0.1:34583 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,706 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-291656986_22 at /127.0.0.1:41292 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41292 dst: /127.0.0.1:34583 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,706 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-486661064_22 at /127.0.0.1:56484 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56484 dst: /127.0.0.1:44245 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,707 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-291656986_22 at /127.0.0.1:56520 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56520 dst: /127.0.0.1:44245 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,706 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-291656986_22 at /127.0.0.1:56508 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56508 dst: /127.0.0.1:44245 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,707 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-486661064_22 at /127.0.0.1:41272 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41272 dst: /127.0.0.1:34583 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6af8ea6c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:01,709 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26d8ef5e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:01,709 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:01,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69f035ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:01,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a545b17{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:01,710 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:01,710 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:01,710 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:01,710 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid aa17d928-a96e-4537-9725-c53ab17189b7) service to localhost/127.0.0.1:42151 2024-11-11T02:03:01,711 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data1/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:01,711 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data2/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:01,711 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:01,724 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:03:01,731 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:03:01,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:03:01,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:03:01,736 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:03:01,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a69a7b7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:03:01,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@390e479f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:03:01,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@211e6ee7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/java.io.tmpdir/jetty-localhost-33831-hadoop-hdfs-3_4_1-tests_jar-_-any-14823093831949710323/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:01,831 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@52139b2f{HTTP/1.1, (http/1.1)}{localhost:33831} 2024-11-11T02:03:01,831 INFO [Time-limited test {}] server.Server(415): Started @78147ms 2024-11-11T02:03:01,833 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:03:01,851 WARN [AsyncFSWAL-0-hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1-prefix:370bc2ade342,33569,1731290570006 {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,851 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 370bc2ade342%2C33569%2C1731290570006:(num 1731290570426) roll requested 2024-11-11T02:03:01,868 DEBUG [regionserver/370bc2ade342:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290581852, exclude list is [], retry=0 2024-11-11T02:03:01,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45297,DS-c8d575af-798e-4640-9973-f34b52cafee1,DISK] 2024-11-11T02:03:01,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39945,DS-8027192a-9bfb-4d8b-8832-3d1fc3b69bc0,DISK] 2024-11-11T02:03:01,874 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 with entries=3, filesize=2.68 KB; new WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290581852 2024-11-11T02:03:01,874 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44891:44891),(127.0.0.1/127.0.0.1:42283:42283)] 2024-11-11T02:03:01,874 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 is not closed yet, will try archiving it next time 2024-11-11T02:03:01,875 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,875 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:03:01,875 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:03:01,876 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:03:01,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:01,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:01,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:03:01,876 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-11T02:03:01,876 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:03:01,876 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=75559587, stopped=false 2024-11-11T02:03:01,877 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-11T02:03:01,877 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,40297,1731290569962 2024-11-11T02:03:01,877 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 2024-11-11T02:03:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:03:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:03:01,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:01,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:03:01,879 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:03:01,879 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:03:01,879 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:03:01,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:01,879 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,33569,1731290570006' ***** 2024-11-11T02:03:01,879 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:03:01,880 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:03:01,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:03:01,880 INFO [RS:0;370bc2ade342:33569 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:03:01,880 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:03:01,880 INFO [RS:0;370bc2ade342:33569 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:03:01,880 WARN [IPC Server handler 1 on default port 42151 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 has not been closed. Lease recovery is in progress. RecoveryId = 1014 for block blk_1073741833_1009 2024-11-11T02:03:01,880 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(3091): Received CLOSE for 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:03:01,880 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,33569,1731290570006 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:33569. 2024-11-11T02:03:01,881 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7dd8321a8f4ac9f6b1d26f3f16288215, disabling compactions & flushes 2024-11-11T02:03:01,881 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,881 DEBUG [RS:0;370bc2ade342:33569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:03:01,881 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,881 DEBUG [RS:0;370bc2ade342:33569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:01,881 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. after waiting 0 ms 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:03:01,881 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:03:01,881 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7dd8321a8f4ac9f6b1d26f3f16288215 1/1 column families, dataSize=2.05 KB heapSize=2.45 KB 2024-11-11T02:03:01,881 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T02:03:01,881 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7dd8321a8f4ac9f6b1d26f3f16288215=TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215.} 2024-11-11T02:03:01,881 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:03:01,882 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7dd8321a8f4ac9f6b1d26f3f16288215 2024-11-11T02:03:01,882 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:03:01,882 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:03:01,882 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:03:01,882 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:03:01,882 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-11T02:03:01,883 WARN [AsyncFSWAL-0-hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1-prefix:370bc2ade342,33569,1731290570006.meta {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,883 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 370bc2ade342%2C33569%2C1731290570006.meta:.meta(num 1731290570865) roll requested 2024-11-11T02:03:01,884 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 after 5ms 2024-11-11T02:03:01,896 DEBUG [regionserver/370bc2ade342:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290581883.meta, exclude list is [], retry=0 2024-11-11T02:03:01,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44095,DS-8489b8a0-cfdb-46d2-9901-464601b0ac6f,DISK] 2024-11-11T02:03:01,900 INFO [regionserver/370bc2ade342:0.logRoller {}] monitor.ExcludeDatanodeManager(82): Added datanode: DatanodeInfoWithStorage[127.0.0.1:44245,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] to exclude cache by [connect error] success, current excludeDNsCache size=1 2024-11-11T02:03:01,900 WARN [regionserver/370bc2ade342:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(579): create fan-out dfs output /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290581883.meta failed, retry = 0 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: /127.0.0.1:44245 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:01,901 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/.tmp/info/607f92d58ded4d118785133e74ebc362 is 1055, key is row0001/info:/1731290581699/Put/seqid=0 2024-11-11T02:03:01,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741839_1016 (size=7062) 2024-11-11T02:03:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741839_1016 (size=7062) 2024-11-11T02:03:01,913 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.05 KB at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/.tmp/info/607f92d58ded4d118785133e74ebc362 2024-11-11T02:03:01,924 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/.tmp/info/607f92d58ded4d118785133e74ebc362 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/info/607f92d58ded4d118785133e74ebc362 2024-11-11T02:03:01,933 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/info/607f92d58ded4d118785133e74ebc362, entries=2, sequenceid=6, filesize=6.9 K 2024-11-11T02:03:01,934 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.05 KB/2102, heapSize ~2.44 KB/2496, currentSize=0 B/0 for 7dd8321a8f4ac9f6b1d26f3f16288215 in 53ms, sequenceid=6, compaction requested=false 2024-11-11T02:03:01,939 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7dd8321a8f4ac9f6b1d26f3f16288215/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T02:03:01,940 WARN [Thread-608 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:03:01,940 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,940 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7dd8321a8f4ac9f6b1d26f3f16288215: Waiting for close lock at 1731290581881Running coprocessor pre-close hooks at 1731290581881Disabling compacts and flushes for region at 1731290581881Disabling writes for close at 1731290581881Obtaining lock to block concurrent updates at 1731290581881Preparing flush snapshotting stores in 7dd8321a8f4ac9f6b1d26f3f16288215 at 1731290581881Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215., syncing WAL and waiting on mvcc, flushsize=dataSize=2102, getHeapSize=2496, getOffHeapSize=0, getCellsCount=2 at 1731290581882 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. at 1731290581884 (+2 ms)Flushing 7dd8321a8f4ac9f6b1d26f3f16288215/info: creating writer at 1731290581884Flushing 7dd8321a8f4ac9f6b1d26f3f16288215/info: appending metadata at 1731290581900 (+16 ms)Flushing 7dd8321a8f4ac9f6b1d26f3f16288215/info: closing flushed file at 1731290581900Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7127b4ab: reopening flushed file at 1731290581923 (+23 ms)Finished flush of dataSize ~2.05 KB/2102, heapSize ~2.44 KB/2496, currentSize=0 B/0 for 7dd8321a8f4ac9f6b1d26f3f16288215 in 53ms, sequenceid=6, compaction requested=false at 1731290581934 (+11 ms)Writing region close event to WAL at 1731290581935 (+1 ms)Running coprocessor post-close hooks at 1731290581940 (+5 ms)Closed at 1731290581940 2024-11-11T02:03:01,940 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215. 2024-11-11T02:03:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x144f6b7e6a47f9cb with lease ID 0x26506e4ffbc32e51: from storage DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40 node DatanodeRegistration(127.0.0.1:41275, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=44617, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 5, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T02:03:01,944 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x144f6b7e6a47f9cb with lease ID 0x26506e4ffbc32e51: from storage DS-f197bfa4-a279-4ff7-977b-abd6c4a5ba4a node DatanodeRegistration(127.0.0.1:41275, datanodeUuid=aa17d928-a96e-4537-9725-c53ab17189b7, infoPort=44617, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=40305104;c=1731290569374), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:03:02,001 DEBUG [regionserver/370bc2ade342:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290581883.meta, exclude list is [(127.0.0.1/127.0.0.1:42869:42869)], retry=1 2024-11-11T02:03:02,002 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-291656986_22 at /127.0.0.1:35634 [Receiving block BP-1431619133-172.17.0.2-1731290569374:blk_1073741838_1015] {}] datanode.DataXceiver(331): 127.0.0.1:44095:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35634 dst: /127.0.0.1:44095 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:02,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45297,DS-c8d575af-798e-4640-9973-f34b52cafee1,DISK] 2024-11-11T02:03:02,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41275,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] 2024-11-11T02:03:02,010 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta with entries=9, filesize=2.52 KB; new WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290581883.meta 2024-11-11T02:03:02,010 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44891:44891),(127.0.0.1/127.0.0.1:44617:44617)] 2024-11-11T02:03:02,011 DEBUG [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta is not closed yet, will try archiving it next time 2024-11-11T02:03:02,011 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:02,011 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:02,011 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta 2024-11-11T02:03:02,012 WARN [IPC Server handler 2 on default port 42151 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta has not been closed. Lease recovery is in progress. RecoveryId = 1018 for block blk_1073741834_1010 2024-11-11T02:03:02,012 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta after 1ms 2024-11-11T02:03:02,032 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/info/fbc1b3d0a1274c1da7aa79dd46d735cb is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731290571578.7dd8321a8f4ac9f6b1d26f3f16288215./info:regioninfo/1731290571960/Put/seqid=0 2024-11-11T02:03:02,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741841_1019 (size=7089) 2024-11-11T02:03:02,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741841_1019 (size=7089) 2024-11-11T02:03:02,039 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/info/fbc1b3d0a1274c1da7aa79dd46d735cb 2024-11-11T02:03:02,059 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/ns/72073059db2e4b20b6d48f118307493a is 43, key is default/ns:d/1731290570925/Put/seqid=0 2024-11-11T02:03:02,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741842_1020 (size=5153) 2024-11-11T02:03:02,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41275 is added to blk_1073741842_1020 (size=5153) 2024-11-11T02:03:02,066 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/ns/72073059db2e4b20b6d48f118307493a 2024-11-11T02:03:02,082 DEBUG [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:03:02,088 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/table/158c4a0470694958b2074bb648c9b515 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731290571973/Put/seqid=0 2024-11-11T02:03:02,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741843_1021 (size=5424) 2024-11-11T02:03:02,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741843_1021 (size=5424) 2024-11-11T02:03:02,094 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/table/158c4a0470694958b2074bb648c9b515 2024-11-11T02:03:02,103 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/info/fbc1b3d0a1274c1da7aa79dd46d735cb as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/info/fbc1b3d0a1274c1da7aa79dd46d735cb 2024-11-11T02:03:02,111 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/info/fbc1b3d0a1274c1da7aa79dd46d735cb, entries=10, sequenceid=11, filesize=6.9 K 2024-11-11T02:03:02,113 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/ns/72073059db2e4b20b6d48f118307493a as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/ns/72073059db2e4b20b6d48f118307493a 2024-11-11T02:03:02,122 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/ns/72073059db2e4b20b6d48f118307493a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-11T02:03:02,123 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/.tmp/table/158c4a0470694958b2074bb648c9b515 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/table/158c4a0470694958b2074bb648c9b515 2024-11-11T02:03:02,131 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/table/158c4a0470694958b2074bb648c9b515, entries=2, sequenceid=11, filesize=5.3 K 2024-11-11T02:03:02,133 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=11, compaction requested=false 2024-11-11T02:03:02,138 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-11T02:03:02,139 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:03:02,139 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:03:02,140 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290581881Running coprocessor pre-close hooks at 1731290581881Disabling compacts and flushes for region at 1731290581881Disabling writes for close at 1731290581882 (+1 ms)Obtaining lock to block concurrent updates at 1731290581882Preparing flush snapshotting stores in 1588230740 at 1731290581882Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731290581883 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731290582011 (+128 ms)Flushing 1588230740/info: creating writer at 1731290582011Flushing 1588230740/info: appending metadata at 1731290582032 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731290582032Flushing 1588230740/ns: creating writer at 1731290582044 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731290582059 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731290582059Flushing 1588230740/table: creating writer at 1731290582073 (+14 ms)Flushing 1588230740/table: appending metadata at 1731290582087 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731290582088 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17a6a0fc: reopening flushed file at 1731290582101 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42eb7398: reopening flushed file at 1731290582111 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@776a948: reopening flushed file at 1731290582122 (+11 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=11, compaction requested=false at 1731290582133 (+11 ms)Writing region close event to WAL at 1731290582134 (+1 ms)Running coprocessor post-close hooks at 1731290582139 (+5 ms)Closed at 1731290582139 2024-11-11T02:03:02,140 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:03:02,278 INFO [regionserver/370bc2ade342:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:03:02,282 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,33569,1731290570006; all regions closed. 2024-11-11T02:03:02,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741840_1017 (size=825) 2024-11-11T02:03:02,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41275 is added to blk_1073741840_1017 (size=825) 2024-11-11T02:03:02,376 INFO [regionserver/370bc2ade342:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T02:03:02,376 INFO [regionserver/370bc2ade342:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T02:03:04,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741834_1018 (size=2386) 2024-11-11T02:03:05,887 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 after 4009ms 2024-11-11T02:03:06,013 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta after 4002ms 2024-11-11T02:03:06,014 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta to hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs/370bc2ade342%2C33569%2C1731290570006.meta.1731290570865.meta 2024-11-11T02:03:06,018 DEBUG [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs 2024-11-11T02:03:06,018 INFO [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C33569%2C1731290570006.meta:.meta(num 1731290581883) 2024-11-11T02:03:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741837_1013 (size=2288) 2024-11-11T02:03:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741837_1013 (size=2288) 2024-11-11T02:03:06,942 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:06,962 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:06,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:07,474 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:03:07,479 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:07,499 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:07,500 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:07,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741834_1018 (size=2386) 2024-11-11T02:03:08,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-11T02:03:08,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:03:08,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:03:11,019 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.async.wait.on.shutdown.seconds" 2024-11-11T02:03:11,029 DEBUG [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs 2024-11-11T02:03:11,029 INFO [RS:0;370bc2ade342:33569 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C33569%2C1731290570006:(num 1731290581852) 2024-11-11T02:03:11,029 DEBUG [RS:0;370bc2ade342:33569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:11,029 INFO [RS:0;370bc2ade342:33569 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:03:11,029 INFO [RS:0;370bc2ade342:33569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:03:11,030 INFO [RS:0;370bc2ade342:33569 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T02:03:11,030 INFO [RS:0;370bc2ade342:33569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:03:11,030 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:03:11,031 INFO [RS:0;370bc2ade342:33569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33569 2024-11-11T02:03:11,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:03:11,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,33569,1731290570006 2024-11-11T02:03:11,032 INFO [RS:0;370bc2ade342:33569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:03:11,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,33569,1731290570006] 2024-11-11T02:03:11,034 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,33569,1731290570006 already deleted, retry=false 2024-11-11T02:03:11,035 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,33569,1731290570006 expired; onlineServers=0 2024-11-11T02:03:11,035 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,40297,1731290569962' ***** 2024-11-11T02:03:11,035 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:03:11,035 INFO [M:0;370bc2ade342:40297 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:03:11,035 INFO [M:0;370bc2ade342:40297 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:03:11,035 DEBUG [M:0;370bc2ade342:40297 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:03:11,035 DEBUG [M:0;370bc2ade342:40297 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:03:11,035 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:03:11,035 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290570226 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290570226,5,FailOnTimeoutGroup] 2024-11-11T02:03:11,035 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290570228 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290570228,5,FailOnTimeoutGroup] 2024-11-11T02:03:11,036 INFO [M:0;370bc2ade342:40297 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:03:11,036 INFO [M:0;370bc2ade342:40297 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:03:11,036 DEBUG [M:0;370bc2ade342:40297 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:03:11,036 INFO [M:0;370bc2ade342:40297 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:03:11,036 INFO [M:0;370bc2ade342:40297 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:03:11,036 INFO [M:0;370bc2ade342:40297 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:03:11,036 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:03:11,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:03:11,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:11,037 DEBUG [M:0;370bc2ade342:40297 {}] zookeeper.ZKUtil(347): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:03:11,037 WARN [M:0;370bc2ade342:40297 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:03:11,039 INFO [M:0;370bc2ade342:40297 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/.lastflushedseqids 2024-11-11T02:03:11,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741844_1022 (size=139) 2024-11-11T02:03:11,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41275 is added to blk_1073741844_1022 (size=139) 2024-11-11T02:03:11,046 INFO [M:0;370bc2ade342:40297 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:03:11,046 INFO [M:0;370bc2ade342:40297 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:03:11,046 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:03:11,046 INFO [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:11,046 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:11,046 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:03:11,046 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:11,047 INFO [M:0;370bc2ade342:40297 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.09 KB heapSize=29.05 KB 2024-11-11T02:03:11,047 WARN [AsyncFSWAL-0-hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData-prefix:370bc2ade342,40297,1731290569962 {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:11,048 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 370bc2ade342%2C40297%2C1731290569962:(num 1731290570133) roll requested 2024-11-11T02:03:11,062 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290591048, exclude list is [], retry=0 2024-11-11T02:03:11,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39945,DS-8027192a-9bfb-4d8b-8832-3d1fc3b69bc0,DISK] 2024-11-11T02:03:11,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41275,DS-41cfba4b-b94a-4f3f-8ca1-ed0e12ebab40,DISK] 2024-11-11T02:03:11,067 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 with entries=54, filesize=26.80 KB; new WAL /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290591048 2024-11-11T02:03:11,067 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44617:44617),(127.0.0.1/127.0.0.1:42283:42283)] 2024-11-11T02:03:11,067 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 is not closed yet, will try archiving it next time 2024-11-11T02:03:11,067 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:11,067 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:11,067 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 2024-11-11T02:03:11,068 WARN [IPC Server handler 3 on default port 42151 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741830_1006 2024-11-11T02:03:11,068 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 after 1ms 2024-11-11T02:03:11,083 DEBUG [M:0;370bc2ade342:40297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2138969e8f5a4b2c962644b45d2975c5 is 82, key is hbase:meta,,1/info:regioninfo/1731290570908/Put/seqid=0 2024-11-11T02:03:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741846_1025 (size=5672) 2024-11-11T02:03:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741846_1025 (size=5672) 2024-11-11T02:03:11,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:03:11,134 INFO [RS:0;370bc2ade342:33569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:03:11,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33569-0x1002c84f9460001, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:03:11,134 INFO [RS:0;370bc2ade342:33569 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,33569,1731290570006; zookeeper connection closed. 2024-11-11T02:03:11,134 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@da8bb41 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@da8bb41 2024-11-11T02:03:11,135 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:03:11,492 INFO [M:0;370bc2ade342:40297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2138969e8f5a4b2c962644b45d2975c5 2024-11-11T02:03:11,522 DEBUG [M:0;370bc2ade342:40297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/11cefcb4151d493aa3b16b352c62d851 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731290571979/Put/seqid=0 2024-11-11T02:03:11,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741847_1026 (size=6114) 2024-11-11T02:03:11,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741847_1026 (size=6114) 2024-11-11T02:03:11,529 INFO [M:0;370bc2ade342:40297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.49 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/11cefcb4151d493aa3b16b352c62d851 2024-11-11T02:03:11,548 DEBUG [M:0;370bc2ade342:40297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60cd468f2b1d421c997e68b54ce8e478 is 69, key is 370bc2ade342,33569,1731290570006/rs:state/1731290570255/Put/seqid=0 2024-11-11T02:03:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741848_1027 (size=5156) 2024-11-11T02:03:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45297 is added to blk_1073741848_1027 (size=5156) 2024-11-11T02:03:11,554 INFO [M:0;370bc2ade342:40297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60cd468f2b1d421c997e68b54ce8e478 2024-11-11T02:03:11,573 DEBUG [M:0;370bc2ade342:40297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bde41f7cc5f54b3c86bca9c66a9034a6 is 52, key is load_balancer_on/state:d/1731290571051/Put/seqid=0 2024-11-11T02:03:11,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741849_1028 (size=5056) 2024-11-11T02:03:11,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741849_1028 (size=5056) 2024-11-11T02:03:11,578 INFO [M:0;370bc2ade342:40297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bde41f7cc5f54b3c86bca9c66a9034a6 2024-11-11T02:03:11,586 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2138969e8f5a4b2c962644b45d2975c5 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2138969e8f5a4b2c962644b45d2975c5 2024-11-11T02:03:11,593 INFO [M:0;370bc2ade342:40297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2138969e8f5a4b2c962644b45d2975c5, entries=8, sequenceid=56, filesize=5.5 K 2024-11-11T02:03:11,594 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/11cefcb4151d493aa3b16b352c62d851 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/11cefcb4151d493aa3b16b352c62d851 2024-11-11T02:03:11,601 INFO [M:0;370bc2ade342:40297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/11cefcb4151d493aa3b16b352c62d851, entries=6, sequenceid=56, filesize=6.0 K 2024-11-11T02:03:11,603 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60cd468f2b1d421c997e68b54ce8e478 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60cd468f2b1d421c997e68b54ce8e478 2024-11-11T02:03:11,609 INFO [M:0;370bc2ade342:40297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60cd468f2b1d421c997e68b54ce8e478, entries=1, sequenceid=56, filesize=5.0 K 2024-11-11T02:03:11,611 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bde41f7cc5f54b3c86bca9c66a9034a6 as hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bde41f7cc5f54b3c86bca9c66a9034a6 2024-11-11T02:03:11,617 INFO [M:0;370bc2ade342:40297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bde41f7cc5f54b3c86bca9c66a9034a6, entries=1, sequenceid=56, filesize=4.9 K 2024-11-11T02:03:11,618 INFO [M:0;370bc2ade342:40297 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.09 KB/23644, heapSize ~28.98 KB/29680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 572ms, sequenceid=56, compaction requested=false 2024-11-11T02:03:11,620 INFO [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:11,620 DEBUG [M:0;370bc2ade342:40297 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290591046Disabling compacts and flushes for region at 1731290591046Disabling writes for close at 1731290591046Obtaining lock to block concurrent updates at 1731290591047 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731290591047Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23644, getHeapSize=29680, getOffHeapSize=0, getCellsCount=67 at 1731290591047Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731290591067 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731290591067Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731290591082 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731290591082Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731290591507 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731290591522 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731290591522Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731290591535 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731290591547 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731290591548 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731290591559 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731290591572 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731290591572Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f2dfda: reopening flushed file at 1731290591585 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b9bc2d: reopening flushed file at 1731290591593 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e2ad80c: reopening flushed file at 1731290591601 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c33fa35: reopening flushed file at 1731290591609 (+8 ms)Finished flush of dataSize ~23.09 KB/23644, heapSize ~28.98 KB/29680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 572ms, sequenceid=56, compaction requested=false at 1731290591618 (+9 ms)Writing region close event to WAL at 1731290591620 (+2 ms)Closed at 1731290591620 2024-11-11T02:03:11,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741845_1023 (size=757) 2024-11-11T02:03:11,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41275 is added to blk_1073741845_1023 (size=757) 2024-11-11T02:03:11,903 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-11T02:03:11,905 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.InterruptedIOException: Operation cancelled at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.checkIfCancelled(RecoverLeaseFSUtils.java:269) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:190) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:11,906 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 2024-11-11T02:03:11,906 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=0 on file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 after 0ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:11,915 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@653023e4 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1431619133-172.17.0.2-1731290569374:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:44245,null,null]) java.net.ConnectException: Call From 370bc2ade342/172.17.0.2 to localhost:32823 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:13,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39945 is added to blk_1073741833_1014 (size=1599) 2024-11-11T02:03:13,961 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:03:13,964 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:13,983 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:13,983 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:15,070 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 after 4003ms 2024-11-11T02:03:15,072 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/WALs/370bc2ade342,40297,1731290569962/370bc2ade342%2C40297%2C1731290569962.1731290570133 to hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/oldWALs/370bc2ade342%2C40297%2C1731290569962.1731290570133 2024-11-11T02:03:15,079 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/MasterData/oldWALs/370bc2ade342%2C40297%2C1731290569962.1731290570133 to hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/oldWALs/370bc2ade342%2C40297%2C1731290569962.1731290570133$masterlocalwal$ 2024-11-11T02:03:15,080 INFO [M:0;370bc2ade342:40297 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:03:15,080 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:03:15,080 INFO [M:0;370bc2ade342:40297 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40297 2024-11-11T02:03:15,080 INFO [M:0;370bc2ade342:40297 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:03:15,182 INFO [M:0;370bc2ade342:40297 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:03:15,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:03:15,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40297-0x1002c84f9460000, quorum=127.0.0.1:63037, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:03:15,188 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@211e6ee7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,189 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@52139b2f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,189 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@390e479f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a69a7b7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,192 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:15,192 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:15,192 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid aa17d928-a96e-4537-9725-c53ab17189b7) service to localhost/127.0.0.1:42151 2024-11-11T02:03:15,192 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:15,193 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data1/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,193 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data2/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,193 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:15,202 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7016180d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,202 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64bc4e58{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,202 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,203 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b434fa5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,203 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63f1466f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,204 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:15,204 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:15,204 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid c12e3151-19d6-4d77-93d4-5d8e123bcf58) service to localhost/127.0.0.1:42151 2024-11-11T02:03:15,204 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:15,205 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data9/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,205 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data10/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,205 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:15,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a5edbcb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,210 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@177b2eb8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,210 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c1a1532{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f965db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,211 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:15,211 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:15,211 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid d4317a7e-6ca3-4230-984c-daafe1b742f0) service to localhost/127.0.0.1:42151 2024-11-11T02:03:15,211 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:15,212 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data7/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,212 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data8/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,212 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:15,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59d8e30e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,214 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1d2cd94{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,214 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59327d46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6299fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,215 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:15,215 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:15,215 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:15,215 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid ffa2cb22-3a7a-4e70-a5ac-c66cebd206fd) service to localhost/127.0.0.1:42151 2024-11-11T02:03:15,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data5/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data6/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,216 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:15,221 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a58fc3f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,221 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f06258f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,221 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5047de0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26c15ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,223 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:03:15,223 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:03:15,223 WARN [BP-1431619133-172.17.0.2-1731290569374 heartbeating to localhost/127.0.0.1:42151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1431619133-172.17.0.2-1731290569374 (Datanode Uuid 8c86de0f-bd6f-4f7f-9492-2ba399dee161) service to localhost/127.0.0.1:42151 2024-11-11T02:03:15,223 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:03:15,224 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data3/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,224 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/cluster_ad1772df-eef7-cd86-86a1-0d53c7b32ea0/data/data4/current/BP-1431619133-172.17.0.2-1731290569374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:03:15,224 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:03:15,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@375d1d1a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:03:15,230 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@487bdcb9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:03:15,230 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:03:15,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fba372a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:03:15,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7648fb82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir/,STOPPED} 2024-11-11T02:03:15,237 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:03:15,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T02:03:15,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRollOnDatanodeDeath Thread=102 (was 66) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:42151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:42151 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-11-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:42151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:166) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$908/0x00007fc724bfcff8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:42151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:42151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=437 (was 390) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=58 (was 41) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6234 (was 6524) 2024-11-11T02:03:15,281 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testCompactionRecordDoesntBlockRolling Thread=102, OpenFileDescriptor=437, MaxFileDescriptor=1048576, SystemLoadAverage=58, ProcessCount=11, AvailableMemoryMB=6234 2024-11-11T02:03:15,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:03:15,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.log.dir so I do NOT create it in target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7 2024-11-11T02:03:15,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1ad9b94-6ce6-e106-e8bd-439fe7dd5d32/hadoop.tmp.dir so I do NOT create it in target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e, deleteOnExit=true 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/test.cache.data in system properties and HBase conf 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:03:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:03:15,282 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:03:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:03:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:03:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:03:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:03:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:03:15,345 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:03:15,350 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:03:15,351 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:03:15,351 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:03:15,351 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:03:15,352 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:03:15,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@372de30f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:03:15,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57aab5a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:03:15,443 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6b4d832e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/java.io.tmpdir/jetty-localhost-43473-hadoop-hdfs-3_4_1-tests_jar-_-any-16685264035882357695/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:03:15,444 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@147f88bb{HTTP/1.1, (http/1.1)}{localhost:43473} 2024-11-11T02:03:15,444 INFO [Time-limited test {}] server.Server(415): Started @91760ms 2024-11-11T02:03:15,527 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:03:15,530 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:03:15,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:03:15,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:03:15,531 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:03:15,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a5885a7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:03:15,532 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@105345dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:03:15,629 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c3d0a48{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/java.io.tmpdir/jetty-localhost-39745-hadoop-hdfs-3_4_1-tests_jar-_-any-6277618212264971829/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,630 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@784f9787{HTTP/1.1, (http/1.1)}{localhost:39745} 2024-11-11T02:03:15,630 INFO [Time-limited test {}] server.Server(415): Started @91946ms 2024-11-11T02:03:15,631 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:03:15,656 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:03:15,659 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:03:15,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:03:15,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:03:15,659 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:03:15,660 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31c3a914{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:03:15,660 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38d7638f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:03:15,685 WARN [Thread-788 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data2/current/BP-771446984-172.17.0.2-1731290595307/current, will proceed with Du for space computation calculation, 2024-11-11T02:03:15,685 WARN [Thread-787 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data1/current/BP-771446984-172.17.0.2-1731290595307/current, will proceed with Du for space computation calculation, 2024-11-11T02:03:15,699 WARN [Thread-766 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:03:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51c5a511a71ffe42 with lease ID 0xe3e7d94ce4ce8601: Processing first storage report for DS-642ca475-a9fd-4327-ba95-1c7aa4047d26 from datanode DatanodeRegistration(127.0.0.1:34449, datanodeUuid=8836dfa1-1616-448b-880a-a9728d93ef8d, infoPort=35029, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307) 2024-11-11T02:03:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51c5a511a71ffe42 with lease ID 0xe3e7d94ce4ce8601: from storage DS-642ca475-a9fd-4327-ba95-1c7aa4047d26 node DatanodeRegistration(127.0.0.1:34449, datanodeUuid=8836dfa1-1616-448b-880a-a9728d93ef8d, infoPort=35029, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:03:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51c5a511a71ffe42 with lease ID 0xe3e7d94ce4ce8601: Processing first storage report for DS-8a66d0e8-ca81-4b79-ac1a-320f95d9d194 from datanode DatanodeRegistration(127.0.0.1:34449, datanodeUuid=8836dfa1-1616-448b-880a-a9728d93ef8d, infoPort=35029, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307) 2024-11-11T02:03:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51c5a511a71ffe42 with lease ID 0xe3e7d94ce4ce8601: from storage DS-8a66d0e8-ca81-4b79-ac1a-320f95d9d194 node DatanodeRegistration(127.0.0.1:34449, datanodeUuid=8836dfa1-1616-448b-880a-a9728d93ef8d, infoPort=35029, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:03:15,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d79aa7b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/java.io.tmpdir/jetty-localhost-46217-hadoop-hdfs-3_4_1-tests_jar-_-any-15904900744407469598/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:03:15,756 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3af67f5f{HTTP/1.1, (http/1.1)}{localhost:46217} 2024-11-11T02:03:15,756 INFO [Time-limited test {}] server.Server(415): Started @92072ms 2024-11-11T02:03:15,757 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:03:15,811 WARN [Thread-813 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data3/current/BP-771446984-172.17.0.2-1731290595307/current, will proceed with Du for space computation calculation, 2024-11-11T02:03:15,811 WARN [Thread-814 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data4/current/BP-771446984-172.17.0.2-1731290595307/current, will proceed with Du for space computation calculation, 2024-11-11T02:03:15,828 WARN [Thread-802 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:03:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd24de114348309ba with lease ID 0xe3e7d94ce4ce8602: Processing first storage report for DS-96b192bb-4b22-4a34-968f-19e3ce6d6845 from datanode DatanodeRegistration(127.0.0.1:39529, datanodeUuid=4f58aede-1c4f-468d-ad99-38501f969cc6, infoPort=44471, infoSecurePort=0, ipcPort=46563, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307) 2024-11-11T02:03:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd24de114348309ba with lease ID 0xe3e7d94ce4ce8602: from storage DS-96b192bb-4b22-4a34-968f-19e3ce6d6845 node DatanodeRegistration(127.0.0.1:39529, datanodeUuid=4f58aede-1c4f-468d-ad99-38501f969cc6, infoPort=44471, infoSecurePort=0, ipcPort=46563, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:03:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd24de114348309ba with lease ID 0xe3e7d94ce4ce8602: Processing first storage report for DS-28ae874b-f366-456c-89ac-2b46e70c9db8 from datanode DatanodeRegistration(127.0.0.1:39529, datanodeUuid=4f58aede-1c4f-468d-ad99-38501f969cc6, infoPort=44471, infoSecurePort=0, ipcPort=46563, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307) 2024-11-11T02:03:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd24de114348309ba with lease ID 0xe3e7d94ce4ce8602: from storage DS-28ae874b-f366-456c-89ac-2b46e70c9db8 node DatanodeRegistration(127.0.0.1:39529, datanodeUuid=4f58aede-1c4f-468d-ad99-38501f969cc6, infoPort=44471, infoSecurePort=0, ipcPort=46563, storageInfo=lv=-57;cid=testClusterID;nsid=220329827;c=1731290595307), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:03:15,879 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7 2024-11-11T02:03:15,884 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/zookeeper_0, clientPort=58132, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:03:15,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58132 2024-11-11T02:03:15,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:03:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:03:15,899 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb with version=8 2024-11-11T02:03:15,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase-staging 2024-11-11T02:03:15,902 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:03:15,902 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:03:15,903 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:03:15,903 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41829 2024-11-11T02:03:15,905 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41829 connecting to ZooKeeper ensemble=127.0.0.1:58132 2024-11-11T02:03:15,908 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=1 on file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 after 4001ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:03:15,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418290x0, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:03:15,909 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41829-0x1002c855e980000 connected 2024-11-11T02:03:15,923 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,925 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:03:15,927 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb, hbase.cluster.distributed=false 2024-11-11T02:03:15,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:03:15,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-11T02:03:15,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41829 2024-11-11T02:03:15,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41829 2024-11-11T02:03:15,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-11T02:03:15,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41829 2024-11-11T02:03:15,943 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:03:15,943 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:03:15,944 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40699 2024-11-11T02:03:15,945 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40699 connecting to ZooKeeper ensemble=127.0.0.1:58132 2024-11-11T02:03:15,946 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,947 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:15,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406990x0, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:03:15,951 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:03:15,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40699-0x1002c855e980001 connected 2024-11-11T02:03:15,951 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:03:15,952 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:03:15,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:03:15,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:03:15,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40699 2024-11-11T02:03:15,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40699 2024-11-11T02:03:15,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40699 2024-11-11T02:03:15,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40699 2024-11-11T02:03:15,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40699 2024-11-11T02:03:15,971 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:41829 2024-11-11T02:03:15,972 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,41829,1731290595901 2024-11-11T02:03:15,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:03:15,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:03:15,973 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,41829,1731290595901 2024-11-11T02:03:15,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:03:15,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:15,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:15,975 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:03:15,975 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,41829,1731290595901 from backup master directory 2024-11-11T02:03:15,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,41829,1731290595901 2024-11-11T02:03:15,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:03:15,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:03:15,976 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:03:15,976 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,41829,1731290595901 2024-11-11T02:03:15,984 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/hbase.id] with ID: fd7e8f6b-a9c8-4459-b521-0d6db5d94518 2024-11-11T02:03:15,984 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/.tmp/hbase.id 2024-11-11T02:03:15,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:03:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:03:15,990 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/.tmp/hbase.id]:[hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/hbase.id] 2024-11-11T02:03:16,003 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:16,003 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:03:16,005 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-11T02:03:16,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:03:16,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:03:16,014 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:03:16,015 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:03:16,016 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:03:16,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:03:16,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:03:16,428 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store 2024-11-11T02:03:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:03:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:03:16,442 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:16,442 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:03:16,442 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:16,442 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:16,442 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:03:16,442 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:16,442 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:03:16,443 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290596442Disabling compacts and flushes for region at 1731290596442Disabling writes for close at 1731290596442Writing region close event to WAL at 1731290596442Closed at 1731290596442 2024-11-11T02:03:16,444 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/.initializing 2024-11-11T02:03:16,444 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/WALs/370bc2ade342,41829,1731290595901 2024-11-11T02:03:16,445 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:03:16,448 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C41829%2C1731290595901, suffix=, logDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/WALs/370bc2ade342,41829,1731290595901, archiveDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/oldWALs, maxLogs=10 2024-11-11T02:03:16,459 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/WALs/370bc2ade342,41829,1731290595901/370bc2ade342%2C41829%2C1731290595901.1731290596448, exclude list is [], retry=0 2024-11-11T02:03:16,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:03:16,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:03:16,464 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/WALs/370bc2ade342,41829,1731290595901/370bc2ade342%2C41829%2C1731290595901.1731290596448 2024-11-11T02:03:16,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44471:44471),(127.0.0.1/127.0.0.1:35029:35029)] 2024-11-11T02:03:16,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:03:16,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:16,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:03:16,468 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:16,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:03:16,470 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:03:16,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:03:16,472 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:03:16,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:03:16,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:03:16,474 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,477 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,477 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,477 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:03:16,479 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:03:16,481 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:03:16,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698952, jitterRate=-0.11123743653297424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:03:16,482 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731290596465Initializing all the Stores at 1731290596466 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290596466Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290596467 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290596467Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290596467Cleaning up temporary data from old regions at 1731290596477 (+10 ms)Region opened successfully at 1731290596482 (+5 ms) 2024-11-11T02:03:16,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:03:16,486 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5567c9fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:03:16,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:03:16,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:03:16,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:03:16,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:03:16,488 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T02:03:16,489 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T02:03:16,489 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:03:16,491 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:03:16,492 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:03:16,493 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:03:16,493 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:03:16,494 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:03:16,495 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:03:16,495 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:03:16,496 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:03:16,497 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:03:16,498 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:03:16,498 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:03:16,500 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:03:16,501 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:03:16,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:03:16,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:03:16,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,502 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,41829,1731290595901, sessionid=0x1002c855e980000, setting cluster-up flag (Was=false) 2024-11-11T02:03:16,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,507 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:03:16,507 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,41829,1731290595901 2024-11-11T02:03:16,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:16,512 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:03:16,513 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,41829,1731290595901 2024-11-11T02:03:16,514 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:03:16,516 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:03:16,517 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:03:16,517 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:03:16,517 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,41829,1731290595901 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:03:16,519 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,521 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:03:16,521 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:03:16,522 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,522 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:03:16,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731290626525 2024-11-11T02:03:16,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:03:16,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:03:16,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:03:16,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:03:16,526 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:03:16,526 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:03:16,529 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,530 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:03:16,530 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:03:16,530 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:03:16,533 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:03:16,533 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:03:16,533 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290596533,5,FailOnTimeoutGroup] 2024-11-11T02:03:16,533 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290596533,5,FailOnTimeoutGroup] 2024-11-11T02:03:16,533 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,533 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:03:16,534 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,534 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:03:16,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:03:16,564 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(746): ClusterId : fd7e8f6b-a9c8-4459-b521-0d6db5d94518 2024-11-11T02:03:16,564 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:03:16,566 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:03:16,566 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:03:16,567 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:03:16,568 DEBUG [RS:0;370bc2ade342:40699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5891d78d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:03:16,579 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:40699 2024-11-11T02:03:16,579 INFO [RS:0;370bc2ade342:40699 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:03:16,579 INFO [RS:0;370bc2ade342:40699 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:03:16,579 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:03:16,580 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,41829,1731290595901 with port=40699, startcode=1731290595943 2024-11-11T02:03:16,580 DEBUG [RS:0;370bc2ade342:40699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:03:16,582 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46107, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:03:16,583 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41829 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,583 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41829 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,585 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb 2024-11-11T02:03:16,585 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39347 2024-11-11T02:03:16,585 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:03:16,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:03:16,587 DEBUG [RS:0;370bc2ade342:40699 {}] zookeeper.ZKUtil(111): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,587 WARN [RS:0;370bc2ade342:40699 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:03:16,587 INFO [RS:0;370bc2ade342:40699 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:03:16,587 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,587 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,40699,1731290595943] 2024-11-11T02:03:16,596 INFO [RS:0;370bc2ade342:40699 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:03:16,598 INFO [RS:0;370bc2ade342:40699 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:03:16,599 INFO [RS:0;370bc2ade342:40699 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:03:16,599 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,599 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:03:16,600 INFO [RS:0;370bc2ade342:40699 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:03:16,600 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,600 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:03:16,601 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:03:16,602 DEBUG [RS:0;370bc2ade342:40699 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,602 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40699,1731290595943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:03:16,615 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:03:16,615 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40699,1731290595943-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,616 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,616 INFO [RS:0;370bc2ade342:40699 {}] regionserver.Replication(171): 370bc2ade342,40699,1731290595943 started 2024-11-11T02:03:16,628 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:16,628 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,40699,1731290595943, RpcServer on 370bc2ade342/172.17.0.2:40699, sessionid=0x1002c855e980001 2024-11-11T02:03:16,628 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:03:16,628 DEBUG [RS:0;370bc2ade342:40699 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,628 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,40699,1731290595943' 2024-11-11T02:03:16,629 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:03:16,629 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,40699,1731290595943 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,40699,1731290595943' 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:03:16,630 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:03:16,631 DEBUG [RS:0;370bc2ade342:40699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:03:16,631 INFO [RS:0;370bc2ade342:40699 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:03:16,631 INFO [RS:0;370bc2ade342:40699 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:03:16,732 INFO [RS:0;370bc2ade342:40699 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:03:16,735 INFO [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C40699%2C1731290595943, suffix=, logDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943, archiveDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs, maxLogs=32 2024-11-11T02:03:16,753 DEBUG [RS:0;370bc2ade342:40699 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290596737, exclude list is [], retry=0 2024-11-11T02:03:16,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:03:16,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:03:16,758 INFO [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290596737 2024-11-11T02:03:16,758 DEBUG [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44471:44471),(127.0.0.1/127.0.0.1:35029:35029)] 2024-11-11T02:03:16,908 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:16,940 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:03:16,941 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb 2024-11-11T02:03:16,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741833_1009 (size=32) 2024-11-11T02:03:16,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741833_1009 (size=32) 2024-11-11T02:03:16,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:16,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:03:16,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:03:16,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:16,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:03:16,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:03:16,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:16,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:03:16,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:03:16,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:16,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:03:16,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:03:16,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:16,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:16,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:03:16,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740 2024-11-11T02:03:16,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740 2024-11-11T02:03:16,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:03:16,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:03:16,969 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:03:16,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:03:16,972 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:03:16,973 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=716557, jitterRate=-0.08885100483894348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:03:16,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731290596955Initializing all the Stores at 1731290596956 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290596956Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290596956Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290596956Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290596956Cleaning up temporary data from old regions at 1731290596969 (+13 ms)Region opened successfully at 1731290596973 (+4 ms) 2024-11-11T02:03:16,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:03:16,973 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:03:16,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:03:16,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:03:16,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:03:16,974 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:03:16,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290596973Disabling compacts and flushes for region at 1731290596973Disabling writes for close at 1731290596973Writing region close event to WAL at 1731290596974 (+1 ms)Closed at 1731290596974 2024-11-11T02:03:16,975 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:03:16,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:03:16,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:03:16,977 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:03:16,978 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:03:17,128 DEBUG [370bc2ade342:41829 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:03:17,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,40699,1731290595943 2024-11-11T02:03:17,130 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,40699,1731290595943, state=OPENING 2024-11-11T02:03:17,132 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:03:17,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:17,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:03:17,134 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:03:17,134 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:03:17,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,40699,1731290595943}] 2024-11-11T02:03:17,134 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:03:17,291 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:03:17,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:03:17,305 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:03:17,305 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:03:17,305 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:03:17,307 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C40699%2C1731290595943.meta, suffix=.meta, logDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943, archiveDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs, maxLogs=32 2024-11-11T02:03:17,320 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.meta.1731290597308.meta, exclude list is [], retry=0 2024-11-11T02:03:17,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:03:17,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:03:17,324 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.meta.1731290597308.meta 2024-11-11T02:03:17,325 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44471:44471),(127.0.0.1/127.0.0.1:35029:35029)] 2024-11-11T02:03:17,325 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:03:17,325 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:03:17,325 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:03:17,325 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:03:17,325 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:03:17,326 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:17,326 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:03:17,326 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:03:17,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:03:17,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:03:17,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:17,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:03:17,329 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:03:17,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:17,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:03:17,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:03:17,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:17,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:03:17,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:03:17,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:03:17,332 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:03:17,333 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740 2024-11-11T02:03:17,334 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740 2024-11-11T02:03:17,335 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:03:17,335 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:03:17,335 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:03:17,337 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:03:17,337 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812990, jitterRate=0.0337710976600647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:03:17,338 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:03:17,338 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731290597326Writing region info on filesystem at 1731290597326Initializing all the Stores at 1731290597327 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290597327Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290597327Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290597327Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290597327Cleaning up temporary data from old regions at 1731290597335 (+8 ms)Running coprocessor post-open hooks at 1731290597338 (+3 ms)Region opened successfully at 1731290597338 2024-11-11T02:03:17,339 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731290597290 2024-11-11T02:03:17,342 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:03:17,342 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:03:17,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,40699,1731290595943 2024-11-11T02:03:17,343 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,40699,1731290595943, state=OPEN 2024-11-11T02:03:17,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:03:17,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:03:17,346 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,40699,1731290595943 2024-11-11T02:03:17,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:03:17,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:03:17,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:03:17,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,40699,1731290595943 in 212 msec 2024-11-11T02:03:17,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:03:17,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 373 msec 2024-11-11T02:03:17,352 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:03:17,352 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:03:17,353 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:03:17,353 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,40699,1731290595943, seqNum=-1] 2024-11-11T02:03:17,354 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:03:17,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60523, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:03:17,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 843 msec 2024-11-11T02:03:17,360 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731290597360, completionTime=-1 2024-11-11T02:03:17,360 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:03:17,360 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731290657363 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731290717363 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,363 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:41829, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,364 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,364 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,366 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.392sec 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:03:17,368 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:03:17,371 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:03:17,371 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:03:17,371 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,41829,1731290595901-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:17,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551772a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:03:17,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,41829,-1 for getting cluster id 2024-11-11T02:03:17,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:03:17,470 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fd7e8f6b-a9c8-4459-b521-0d6db5d94518' 2024-11-11T02:03:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:03:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fd7e8f6b-a9c8-4459-b521-0d6db5d94518" 2024-11-11T02:03:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc80455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:03:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,41829,-1] 2024-11-11T02:03:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:03:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:03:17,475 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:03:17,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e0f13d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:03:17,476 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:03:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,40699,1731290595943, seqNum=-1] 2024-11-11T02:03:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:03:17,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:03:17,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,41829,1731290595901 2024-11-11T02:03:17,483 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:03:17,485 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-11T02:03:17,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:03:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,41829,1731290595901 2024-11-11T02:03:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47833277 2024-11-11T02:03:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:03:17,489 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:03:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-11T02:03:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-11T02:03:17,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:03:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:17,492 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:03:17,493 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-11T02:03:17,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:03:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:03:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741835_1011 (size=405) 2024-11-11T02:03:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741835_1011 (size=405) 2024-11-11T02:03:17,502 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c559a0f2dc49520aea442fdd2c9cf44, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb 2024-11-11T02:03:17,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741836_1012 (size=88) 2024-11-11T02:03:17,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741836_1012 (size=88) 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1c559a0f2dc49520aea442fdd2c9cf44, disabling compactions & flushes 2024-11-11T02:03:17,509 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. after waiting 0 ms 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,509 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c559a0f2dc49520aea442fdd2c9cf44: Waiting for close lock at 1731290597509Disabling compacts and flushes for region at 1731290597509Disabling writes for close at 1731290597509Writing region close event to WAL at 1731290597509Closed at 1731290597509 2024-11-11T02:03:17,511 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:03:17,511 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731290597511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290597511"}]},"ts":"1731290597511"} 2024-11-11T02:03:17,513 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:03:17,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:03:17,515 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290597514"}]},"ts":"1731290597514"} 2024-11-11T02:03:17,517 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-11T02:03:17,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1c559a0f2dc49520aea442fdd2c9cf44, ASSIGN}] 2024-11-11T02:03:17,518 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1c559a0f2dc49520aea442fdd2c9cf44, ASSIGN 2024-11-11T02:03:17,519 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1c559a0f2dc49520aea442fdd2c9cf44, ASSIGN; state=OFFLINE, location=370bc2ade342,40699,1731290595943; forceNewPlan=false, retain=false 2024-11-11T02:03:17,671 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c559a0f2dc49520aea442fdd2c9cf44, regionState=OPENING, regionLocation=370bc2ade342,40699,1731290595943 2024-11-11T02:03:17,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1c559a0f2dc49520aea442fdd2c9cf44, ASSIGN because future has completed 2024-11-11T02:03:17,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c559a0f2dc49520aea442fdd2c9cf44, server=370bc2ade342,40699,1731290595943}] 2024-11-11T02:03:17,842 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,842 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c559a0f2dc49520aea442fdd2c9cf44, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:03:17,843 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,843 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:03:17,843 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,843 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,845 INFO [StoreOpener-1c559a0f2dc49520aea442fdd2c9cf44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,847 INFO [StoreOpener-1c559a0f2dc49520aea442fdd2c9cf44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c559a0f2dc49520aea442fdd2c9cf44 columnFamilyName info 2024-11-11T02:03:17,847 DEBUG [StoreOpener-1c559a0f2dc49520aea442fdd2c9cf44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:03:17,847 INFO [StoreOpener-1c559a0f2dc49520aea442fdd2c9cf44-1 {}] regionserver.HStore(327): Store=1c559a0f2dc49520aea442fdd2c9cf44/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:03:17,848 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,849 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,849 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,850 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,850 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,852 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,855 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:03:17,855 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1c559a0f2dc49520aea442fdd2c9cf44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755442, jitterRate=-0.03940586745738983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:03:17,855 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:03:17,856 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1c559a0f2dc49520aea442fdd2c9cf44: Running coprocessor pre-open hook at 1731290597843Writing region info on filesystem at 1731290597843Initializing all the Stores at 1731290597844 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290597844Cleaning up temporary data from old regions at 1731290597850 (+6 ms)Running coprocessor post-open hooks at 1731290597855 (+5 ms)Region opened successfully at 1731290597856 (+1 ms) 2024-11-11T02:03:17,857 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44., pid=6, masterSystemTime=1731290597833 2024-11-11T02:03:17,860 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,860 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:17,861 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c559a0f2dc49520aea442fdd2c9cf44, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,40699,1731290595943 2024-11-11T02:03:17,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c559a0f2dc49520aea442fdd2c9cf44, server=370bc2ade342,40699,1731290595943 because future has completed 2024-11-11T02:03:17,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T02:03:17,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c559a0f2dc49520aea442fdd2c9cf44, server=370bc2ade342,40699,1731290595943 in 187 msec 2024-11-11T02:03:17,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T02:03:17,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1c559a0f2dc49520aea442fdd2c9cf44, ASSIGN in 353 msec 2024-11-11T02:03:17,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:03:17,874 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290597873"}]},"ts":"1731290597873"} 2024-11-11T02:03:17,876 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-11T02:03:17,877 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:03:17,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 387 msec 2024-11-11T02:03:17,910 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:18,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:03:18,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T02:03:18,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:03:18,455 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T02:03:18,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:18,455 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-11T02:03:18,911 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:19,912 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:20,913 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:21,914 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:22,832 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:03:22,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,852 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:03:22,868 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:03:22,869 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-11T02:03:22,915 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:23,916 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:24,918 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:25,920 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-11T02:03:26,921 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:03:27,583 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-11T02:03:27,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-11T02:03:27,591 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:27,591 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44., hostname=370bc2ade342,40699,1731290595943, seqNum=2] 2024-11-11T02:03:27,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:27,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:03:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:03:27,611 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:03:27,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:03:27,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40699 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T02:03:27,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:27,783 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1c559a0f2dc49520aea442fdd2c9cf44 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-11T02:03:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/25dab663e9f24db9b019077b1f495b42 is 1055, key is row0001/info:/1731290607598/Put/seqid=0 2024-11-11T02:03:27,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741837_1013 (size=6008) 2024-11-11T02:03:27,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741837_1013 (size=6008) 2024-11-11T02:03:27,804 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/25dab663e9f24db9b019077b1f495b42 2024-11-11T02:03:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/25dab663e9f24db9b019077b1f495b42 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42 2024-11-11T02:03:27,818 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42, entries=1, sequenceid=5, filesize=5.9 K 2024-11-11T02:03:27,819 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 36ms, sequenceid=5, compaction requested=false 2024-11-11T02:03:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1c559a0f2dc49520aea442fdd2c9cf44: 2024-11-11T02:03:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T02:03:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T02:03:27,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T02:03:27,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 212 msec 2024-11-11T02:03:27,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 221 msec 2024-11-11T02:03:27,923 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:28,924 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:29,926 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:30,927 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:31,928 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:32,930 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:33,931 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:34,932 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:35,933 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:36,935 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T02:03:37,631 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-11T02:03:37,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:03:37,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:03:37,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:03:37,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:03:37,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40699 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T02:03:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:37,799 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1c559a0f2dc49520aea442fdd2c9cf44 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-11T02:03:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 is 1055, key is row0002/info:/1731290617633/Put/seqid=0 2024-11-11T02:03:37,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741838_1014 (size=6008) 2024-11-11T02:03:37,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741838_1014 (size=6008) 2024-11-11T02:03:37,815 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 2024-11-11T02:03:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 2024-11-11T02:03:37,831 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66, entries=1, sequenceid=9, filesize=5.9 K 2024-11-11T02:03:37,833 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 33ms, sequenceid=9, compaction requested=false 2024-11-11T02:03:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1c559a0f2dc49520aea442fdd2c9cf44: 2024-11-11T02:03:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T02:03:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T02:03:37,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T02:03:37,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 192 msec 2024-11-11T02:03:37,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 202 msec 2024-11-11T02:03:37,936 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:38,937 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:39,939 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:40,940 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:41,941 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:42,942 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:43,944 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:44,945 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:45,879 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:03:45,947 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:46,948 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T02:03:47,733 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-11T02:03:47,752 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290627739, exclude list is [], retry=0 2024-11-11T02:03:47,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:03:47,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:03:47,756 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290596737 with entries=8, filesize=5.33 KB; new WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290627739 2024-11-11T02:03:47,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35029:35029),(127.0.0.1/127.0.0.1:44471:44471)] 2024-11-11T02:03:47,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290596737 is not closed yet, will try archiving it next time 2024-11-11T02:03:47,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741832_1008 (size=5471) 2024-11-11T02:03:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741832_1008 (size=5471) 2024-11-11T02:03:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:03:47,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:03:47,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:03:47,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:03:47,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40699 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T02:03:47,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:47,916 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1c559a0f2dc49520aea442fdd2c9cf44 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-11T02:03:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/fabdc44cd0a84113b39f2fc26986c019 is 1055, key is row0003/info:/1731290627736/Put/seqid=0 2024-11-11T02:03:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741840_1016 (size=6008) 2024-11-11T02:03:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741840_1016 (size=6008) 2024-11-11T02:03:47,934 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/fabdc44cd0a84113b39f2fc26986c019 2024-11-11T02:03:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/fabdc44cd0a84113b39f2fc26986c019 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019 2024-11-11T02:03:47,949 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019, entries=1, sequenceid=13, filesize=5.9 K 2024-11-11T02:03:47,949 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:47,950 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 34ms, sequenceid=13, compaction requested=true 2024-11-11T02:03:47,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1c559a0f2dc49520aea442fdd2c9cf44: 2024-11-11T02:03:47,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:47,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T02:03:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T02:03:47,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T02:03:47,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 191 msec 2024-11-11T02:03:47,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec 2024-11-11T02:03:48,950 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:49,952 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:50,953 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:51,955 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:52,957 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:53,958 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:54,960 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:55,961 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:56,963 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T02:03:57,862 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-11T02:03:57,867 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:03:57,872 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18024 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:03:57,874 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1c559a0f2dc49520aea442fdd2c9cf44/info is initiating minor compaction (all files) 2024-11-11T02:03:57,874 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:03:57,874 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:03:57,874 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1c559a0f2dc49520aea442fdd2c9cf44/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:57,875 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019] into tmpdir=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp, totalSize=17.6 K 2024-11-11T02:03:57,876 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 25dab663e9f24db9b019077b1f495b42, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731290607598 2024-11-11T02:03:57,876 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aa58bddbe3ec4cd29a19d4d94e0e7e66, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731290617633 2024-11-11T02:03:57,877 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fabdc44cd0a84113b39f2fc26986c019, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731290627736 2024-11-11T02:03:57,898 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1c559a0f2dc49520aea442fdd2c9cf44#info#compaction#21 average throughput is 3.01 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:03:57,899 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/a95eb3ab63f540f59f3c23e9453bdc6b is 1055, key is row0001/info:/1731290607598/Put/seqid=0 2024-11-11T02:03:57,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741841_1017 (size=8221) 2024-11-11T02:03:57,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741841_1017 (size=8221) 2024-11-11T02:03:57,913 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/a95eb3ab63f540f59f3c23e9453bdc6b as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/a95eb3ab63f540f59f3c23e9453bdc6b 2024-11-11T02:03:57,929 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c559a0f2dc49520aea442fdd2c9cf44/info of 1c559a0f2dc49520aea442fdd2c9cf44 into a95eb3ab63f540f59f3c23e9453bdc6b(size=8.0 K), total size for store is 8.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:03:57,929 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1c559a0f2dc49520aea442fdd2c9cf44: 2024-11-11T02:03:57,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290637932, exclude list is [], retry=0 2024-11-11T02:03:57,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:03:57,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:03:57,953 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290627739 with entries=4, filesize=2.43 KB; new WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290637932 2024-11-11T02:03:57,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44471:44471),(127.0.0.1/127.0.0.1:35029:35029)] 2024-11-11T02:03:57,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290627739 is not closed yet, will try archiving it next time 2024-11-11T02:03:57,954 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290596737 to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs/370bc2ade342%2C40699%2C1731290595943.1731290596737 2024-11-11T02:03:57,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:57,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741839_1015 (size=2495) 2024-11-11T02:03:57,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741839_1015 (size=2495) 2024-11-11T02:03:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:03:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:03:57,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-11T02:03:57,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T02:03:57,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T02:03:57,965 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:58,068 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T02:03:58,068 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T02:03:58,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40699 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T02:03:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:58,112 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1c559a0f2dc49520aea442fdd2c9cf44 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-11T02:03:58,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/0899c611701a43fe877e13f41875956b is 1055, key is row0000/info:/1731290637930/Put/seqid=0 2024-11-11T02:03:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741843_1019 (size=6008) 2024-11-11T02:03:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741843_1019 (size=6008) 2024-11-11T02:03:58,523 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/0899c611701a43fe877e13f41875956b 2024-11-11T02:03:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/0899c611701a43fe877e13f41875956b as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/0899c611701a43fe877e13f41875956b 2024-11-11T02:03:58,543 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/0899c611701a43fe877e13f41875956b, entries=1, sequenceid=18, filesize=5.9 K 2024-11-11T02:03:58,544 INFO [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 432ms, sequenceid=18, compaction requested=false 2024-11-11T02:03:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1c559a0f2dc49520aea442fdd2c9cf44: 2024-11-11T02:03:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:03:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T02:03:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T02:03:58,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T02:03:58,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 587 msec 2024-11-11T02:03:58,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 594 msec 2024-11-11T02:03:58,965 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:03:59,966 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:00,967 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:01,969 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:02,843 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1c559a0f2dc49520aea442fdd2c9cf44, had cached 0 bytes from a total of 14229 2024-11-11T02:04:02,971 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:03,972 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:04,974 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:05,976 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:06,977 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:07,978 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41829 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T02:04:08,042 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-11T02:04:08,063 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290648048, exclude list is [], retry=0 2024-11-11T02:04:08,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39529,DS-96b192bb-4b22-4a34-968f-19e3ce6d6845,DISK] 2024-11-11T02:04:08,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34449,DS-642ca475-a9fd-4327-ba95-1c7aa4047d26,DISK] 2024-11-11T02:04:08,067 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290637932 with entries=3, filesize=1.95 KB; new WAL /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290648048 2024-11-11T02:04:08,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44471:44471),(127.0.0.1/127.0.0.1:35029:35029)] 2024-11-11T02:04:08,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290637932 is not closed yet, will try archiving it next time 2024-11-11T02:04:08,067 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/WALs/370bc2ade342,40699,1731290595943/370bc2ade342%2C40699%2C1731290595943.1731290627739 to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs/370bc2ade342%2C40699%2C1731290595943.1731290627739 2024-11-11T02:04:08,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:04:08,067 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:08,067 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:08,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:08,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:08,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:04:08,068 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:04:08,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741842_1018 (size=2001) 2024-11-11T02:04:08,068 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2122453757, stopped=false 2024-11-11T02:04:08,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741842_1018 (size=2001) 2024-11-11T02:04:08,068 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,41829,1731290595901 2024-11-11T02:04:08,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:08,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:08,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:08,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:08,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:04:08,070 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:08,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:08,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:08,070 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,40699,1731290595943' ***** 2024-11-11T02:04:08,070 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:08,070 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:04:08,070 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:08,070 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:04:08,071 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(3091): Received CLOSE for 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,40699,1731290595943 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:40699. 2024-11-11T02:04:08,071 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1c559a0f2dc49520aea442fdd2c9cf44, disabling compactions & flushes 2024-11-11T02:04:08,071 DEBUG [RS:0;370bc2ade342:40699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:08,071 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:04:08,071 DEBUG [RS:0;370bc2ade342:40699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:08,071 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:04:08,071 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. after waiting 0 ms 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:04:08,071 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:04:08,071 INFO [RS:0;370bc2ade342:40699 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:04:08,072 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:04:08,072 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1c559a0f2dc49520aea442fdd2c9cf44 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-11T02:04:08,072 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T02:04:08,072 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1325): Online Regions={1c559a0f2dc49520aea442fdd2c9cf44=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44., 1588230740=hbase:meta,,1.1588230740} 2024-11-11T02:04:08,072 DEBUG [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1c559a0f2dc49520aea442fdd2c9cf44 2024-11-11T02:04:08,072 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:04:08,072 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:04:08,072 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:04:08,072 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:04:08,072 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:04:08,072 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-11T02:04:08,076 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/8abc62c848d14796b00cf98671f5fb0e is 1055, key is row0001/info:/1731290648045/Put/seqid=0 2024-11-11T02:04:08,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741845_1021 (size=6008) 2024-11-11T02:04:08,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741845_1021 (size=6008) 2024-11-11T02:04:08,084 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/8abc62c848d14796b00cf98671f5fb0e 2024-11-11T02:04:08,089 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/info/c7eb58ac53394dfa966c19b4161468c4 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44./info:regioninfo/1731290597861/Put/seqid=0 2024-11-11T02:04:08,093 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/.tmp/info/8abc62c848d14796b00cf98671f5fb0e as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/8abc62c848d14796b00cf98671f5fb0e 2024-11-11T02:04:08,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741846_1022 (size=7308) 2024-11-11T02:04:08,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741846_1022 (size=7308) 2024-11-11T02:04:08,095 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/info/c7eb58ac53394dfa966c19b4161468c4 2024-11-11T02:04:08,099 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/8abc62c848d14796b00cf98671f5fb0e, entries=1, sequenceid=22, filesize=5.9 K 2024-11-11T02:04:08,100 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 29ms, sequenceid=22, compaction requested=true 2024-11-11T02:04:08,101 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019] to archive 2024-11-11T02:04:08,104 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:04:08,106 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42 to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/25dab663e9f24db9b019077b1f495b42 2024-11-11T02:04:08,108 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/aa58bddbe3ec4cd29a19d4d94e0e7e66 2024-11-11T02:04:08,110 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019 to hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/info/fabdc44cd0a84113b39f2fc26986c019 2024-11-11T02:04:08,114 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/ns/792192a9c27c4296ba7ab965a104ed0b is 43, key is default/ns:d/1731290597355/Put/seqid=0 2024-11-11T02:04:08,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741847_1023 (size=5153) 2024-11-11T02:04:08,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741847_1023 (size=5153) 2024-11-11T02:04:08,119 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/ns/792192a9c27c4296ba7ab965a104ed0b 2024-11-11T02:04:08,124 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=370bc2ade342:41829 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-11T02:04:08,124 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [25dab663e9f24db9b019077b1f495b42=6008, aa58bddbe3ec4cd29a19d4d94e0e7e66=6008, fabdc44cd0a84113b39f2fc26986c019=6008] 2024-11-11T02:04:08,128 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1c559a0f2dc49520aea442fdd2c9cf44/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-11T02:04:08,129 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:04:08,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1c559a0f2dc49520aea442fdd2c9cf44: Waiting for close lock at 1731290648071Running coprocessor pre-close hooks at 1731290648071Disabling compacts and flushes for region at 1731290648071Disabling writes for close at 1731290648071Obtaining lock to block concurrent updates at 1731290648072 (+1 ms)Preparing flush snapshotting stores in 1c559a0f2dc49520aea442fdd2c9cf44 at 1731290648072Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44., syncing WAL and waiting on mvcc, flushsize=dataSize=1051, getHeapSize=1368, getOffHeapSize=0, getCellsCount=1 at 1731290648072Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. at 1731290648072Flushing 1c559a0f2dc49520aea442fdd2c9cf44/info: creating writer at 1731290648073 (+1 ms)Flushing 1c559a0f2dc49520aea442fdd2c9cf44/info: appending metadata at 1731290648075 (+2 ms)Flushing 1c559a0f2dc49520aea442fdd2c9cf44/info: closing flushed file at 1731290648075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ae6d7a0: reopening flushed file at 1731290648091 (+16 ms)Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 1c559a0f2dc49520aea442fdd2c9cf44 in 29ms, sequenceid=22, compaction requested=true at 1731290648100 (+9 ms)Writing region close event to WAL at 1731290648125 (+25 ms)Running coprocessor post-close hooks at 1731290648129 (+4 ms)Closed at 1731290648129 2024-11-11T02:04:08,129 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731290597489.1c559a0f2dc49520aea442fdd2c9cf44. 2024-11-11T02:04:08,142 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/table/fa46a767cb3a4e4d8587e88fb285c44e is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731290597873/Put/seqid=0 2024-11-11T02:04:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741848_1024 (size=5508) 2024-11-11T02:04:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741848_1024 (size=5508) 2024-11-11T02:04:08,148 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/table/fa46a767cb3a4e4d8587e88fb285c44e 2024-11-11T02:04:08,155 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/info/c7eb58ac53394dfa966c19b4161468c4 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/info/c7eb58ac53394dfa966c19b4161468c4 2024-11-11T02:04:08,162 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/info/c7eb58ac53394dfa966c19b4161468c4, entries=10, sequenceid=11, filesize=7.1 K 2024-11-11T02:04:08,163 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/ns/792192a9c27c4296ba7ab965a104ed0b as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/ns/792192a9c27c4296ba7ab965a104ed0b 2024-11-11T02:04:08,170 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/ns/792192a9c27c4296ba7ab965a104ed0b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-11T02:04:08,172 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/.tmp/table/fa46a767cb3a4e4d8587e88fb285c44e as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/table/fa46a767cb3a4e4d8587e88fb285c44e 2024-11-11T02:04:08,178 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/table/fa46a767cb3a4e4d8587e88fb285c44e, entries=2, sequenceid=11, filesize=5.4 K 2024-11-11T02:04:08,179 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false 2024-11-11T02:04:08,183 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-11T02:04:08,184 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:08,184 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:08,184 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290648072Running coprocessor pre-close hooks at 1731290648072Disabling compacts and flushes for region at 1731290648072Disabling writes for close at 1731290648072Obtaining lock to block concurrent updates at 1731290648072Preparing flush snapshotting stores in 1588230740 at 1731290648072Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731290648073 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731290648073Flushing 1588230740/info: creating writer at 1731290648073Flushing 1588230740/info: appending metadata at 1731290648089 (+16 ms)Flushing 1588230740/info: closing flushed file at 1731290648089Flushing 1588230740/ns: creating writer at 1731290648100 (+11 ms)Flushing 1588230740/ns: appending metadata at 1731290648113 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731290648113Flushing 1588230740/table: creating writer at 1731290648124 (+11 ms)Flushing 1588230740/table: appending metadata at 1731290648141 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731290648141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1477e6eb: reopening flushed file at 1731290648153 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c7ee45e: reopening flushed file at 1731290648162 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a736e13: reopening flushed file at 1731290648171 (+9 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false at 1731290648179 (+8 ms)Writing region close event to WAL at 1731290648180 (+1 ms)Running coprocessor post-close hooks at 1731290648184 (+4 ms)Closed at 1731290648184 2024-11-11T02:04:08,184 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:08,272 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,40699,1731290595943; all regions closed. 2024-11-11T02:04:08,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741834_1010 (size=3306) 2024-11-11T02:04:08,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741834_1010 (size=3306) 2024-11-11T02:04:08,280 DEBUG [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs 2024-11-11T02:04:08,280 INFO [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C40699%2C1731290595943.meta:.meta(num 1731290597308) 2024-11-11T02:04:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741844_1020 (size=1252) 2024-11-11T02:04:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741844_1020 (size=1252) 2024-11-11T02:04:08,287 DEBUG [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/oldWALs 2024-11-11T02:04:08,287 INFO [RS:0;370bc2ade342:40699 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C40699%2C1731290595943:(num 1731290648048) 2024-11-11T02:04:08,287 DEBUG [RS:0;370bc2ade342:40699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:08,287 INFO [RS:0;370bc2ade342:40699 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:04:08,287 INFO [RS:0;370bc2ade342:40699 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:04:08,287 INFO [RS:0;370bc2ade342:40699 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T02:04:08,288 INFO [RS:0;370bc2ade342:40699 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:04:08,288 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:04:08,288 INFO [RS:0;370bc2ade342:40699 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40699 2024-11-11T02:04:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:04:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,40699,1731290595943 2024-11-11T02:04:08,290 INFO [RS:0;370bc2ade342:40699 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:04:08,290 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,40699,1731290595943] 2024-11-11T02:04:08,291 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,40699,1731290595943 already deleted, retry=false 2024-11-11T02:04:08,291 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,40699,1731290595943 expired; onlineServers=0 2024-11-11T02:04:08,291 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,41829,1731290595901' ***** 2024-11-11T02:04:08,291 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:04:08,291 INFO [M:0;370bc2ade342:41829 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:04:08,291 INFO [M:0;370bc2ade342:41829 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:04:08,291 DEBUG [M:0;370bc2ade342:41829 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:04:08,292 DEBUG [M:0;370bc2ade342:41829 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:04:08,292 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:04:08,292 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290596533 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290596533,5,FailOnTimeoutGroup] 2024-11-11T02:04:08,292 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290596533 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290596533,5,FailOnTimeoutGroup] 2024-11-11T02:04:08,292 INFO [M:0;370bc2ade342:41829 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:04:08,292 INFO [M:0;370bc2ade342:41829 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:04:08,292 DEBUG [M:0;370bc2ade342:41829 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:04:08,292 INFO [M:0;370bc2ade342:41829 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:04:08,292 INFO [M:0;370bc2ade342:41829 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:04:08,293 INFO [M:0;370bc2ade342:41829 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:04:08,293 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:04:08,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:04:08,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:08,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:04:08,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:08,293 DEBUG [M:0;370bc2ade342:41829 {}] zookeeper.ZKUtil(347): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:04:08,293 WARN [M:0;370bc2ade342:41829 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:04:08,295 INFO [M:0;370bc2ade342:41829 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/.lastflushedseqids 2024-11-11T02:04:08,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741849_1025 (size=130) 2024-11-11T02:04:08,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741849_1025 (size=130) 2024-11-11T02:04:08,301 INFO [M:0;370bc2ade342:41829 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:04:08,301 INFO [M:0;370bc2ade342:41829 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:04:08,302 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:04:08,302 INFO [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:08,302 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:08,302 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:04:08,302 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:08,302 INFO [M:0;370bc2ade342:41829 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.56 KB heapSize=54.94 KB 2024-11-11T02:04:08,317 DEBUG [M:0;370bc2ade342:41829 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58282233508e40329714dd27c46dcec6 is 82, key is hbase:meta,,1/info:regioninfo/1731290597342/Put/seqid=0 2024-11-11T02:04:08,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741850_1026 (size=5672) 2024-11-11T02:04:08,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741850_1026 (size=5672) 2024-11-11T02:04:08,322 INFO [M:0;370bc2ade342:41829 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58282233508e40329714dd27c46dcec6 2024-11-11T02:04:08,341 DEBUG [M:0;370bc2ade342:41829 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e978b235b314c7b84897386badbdbce is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731290597878/Put/seqid=0 2024-11-11T02:04:08,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741851_1027 (size=7819) 2024-11-11T02:04:08,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741851_1027 (size=7819) 2024-11-11T02:04:08,346 INFO [M:0;370bc2ade342:41829 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.95 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e978b235b314c7b84897386badbdbce 2024-11-11T02:04:08,350 INFO [M:0;370bc2ade342:41829 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e978b235b314c7b84897386badbdbce 2024-11-11T02:04:08,363 DEBUG [M:0;370bc2ade342:41829 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cca8d27ddbee4f51b6515dec4ae04592 is 69, key is 370bc2ade342,40699,1731290595943/rs:state/1731290596583/Put/seqid=0 2024-11-11T02:04:08,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741852_1028 (size=5156) 2024-11-11T02:04:08,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741852_1028 (size=5156) 2024-11-11T02:04:08,368 INFO [M:0;370bc2ade342:41829 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cca8d27ddbee4f51b6515dec4ae04592 2024-11-11T02:04:08,387 DEBUG [M:0;370bc2ade342:41829 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/302088c7b14e4c2e8c330d67466c065f is 52, key is load_balancer_on/state:d/1731290597484/Put/seqid=0 2024-11-11T02:04:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741853_1029 (size=5056) 2024-11-11T02:04:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741853_1029 (size=5056) 2024-11-11T02:04:08,393 INFO [M:0;370bc2ade342:41829 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/302088c7b14e4c2e8c330d67466c065f 2024-11-11T02:04:08,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:08,394 INFO [RS:0;370bc2ade342:40699 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:04:08,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40699-0x1002c855e980001, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:08,394 INFO [RS:0;370bc2ade342:40699 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,40699,1731290595943; zookeeper connection closed. 2024-11-11T02:04:08,394 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1db8a56 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1db8a56 2024-11-11T02:04:08,394 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:04:08,398 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58282233508e40329714dd27c46dcec6 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58282233508e40329714dd27c46dcec6 2024-11-11T02:04:08,403 INFO [M:0;370bc2ade342:41829 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58282233508e40329714dd27c46dcec6, entries=8, sequenceid=121, filesize=5.5 K 2024-11-11T02:04:08,404 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e978b235b314c7b84897386badbdbce as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e978b235b314c7b84897386badbdbce 2024-11-11T02:04:08,409 INFO [M:0;370bc2ade342:41829 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e978b235b314c7b84897386badbdbce 2024-11-11T02:04:08,409 INFO [M:0;370bc2ade342:41829 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e978b235b314c7b84897386badbdbce, entries=14, sequenceid=121, filesize=7.6 K 2024-11-11T02:04:08,410 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cca8d27ddbee4f51b6515dec4ae04592 as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cca8d27ddbee4f51b6515dec4ae04592 2024-11-11T02:04:08,416 INFO [M:0;370bc2ade342:41829 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cca8d27ddbee4f51b6515dec4ae04592, entries=1, sequenceid=121, filesize=5.0 K 2024-11-11T02:04:08,417 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/302088c7b14e4c2e8c330d67466c065f as hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/302088c7b14e4c2e8c330d67466c065f 2024-11-11T02:04:08,422 INFO [M:0;370bc2ade342:41829 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39347/user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/302088c7b14e4c2e8c330d67466c065f, entries=1, sequenceid=121, filesize=4.9 K 2024-11-11T02:04:08,423 INFO [M:0;370bc2ade342:41829 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=121, compaction requested=false 2024-11-11T02:04:08,425 INFO [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:08,425 DEBUG [M:0;370bc2ade342:41829 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290648302Disabling compacts and flushes for region at 1731290648302Disabling writes for close at 1731290648302Obtaining lock to block concurrent updates at 1731290648302Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731290648302Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44602, getHeapSize=56192, getOffHeapSize=0, getCellsCount=140 at 1731290648302Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731290648303 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731290648303Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731290648317 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731290648317Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731290648327 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731290648340 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731290648340Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731290648350 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731290648363 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731290648363Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731290648373 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731290648387 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731290648387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@327248f6: reopening flushed file at 1731290648397 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@570b458f: reopening flushed file at 1731290648403 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23870f4e: reopening flushed file at 1731290648409 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ac283bd: reopening flushed file at 1731290648416 (+7 ms)Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=121, compaction requested=false at 1731290648423 (+7 ms)Writing region close event to WAL at 1731290648425 (+2 ms)Closed at 1731290648425 2024-11-11T02:04:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34449 is added to blk_1073741830_1006 (size=52999) 2024-11-11T02:04:08,427 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/119da126-bdba-4cf2-0908-c3ee7ab880eb/MasterData/WALs/370bc2ade342,41829,1731290595901/370bc2ade342%2C41829%2C1731290595901.1731290596448 not finished, retry = 0 2024-11-11T02:04:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39529 is added to blk_1073741830_1006 (size=52999) 2024-11-11T02:04:08,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:08,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:04:08,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T02:04:08,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-11T02:04:08,529 INFO [M:0;370bc2ade342:41829 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:04:08,529 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:04:08,529 INFO [M:0;370bc2ade342:41829 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41829 2024-11-11T02:04:08,529 INFO [M:0;370bc2ade342:41829 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:04:08,606 INFO [regionserver/370bc2ade342:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:04:08,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:08,631 INFO [M:0;370bc2ade342:41829 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:04:08,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41829-0x1002c855e980000, quorum=127.0.0.1:58132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:08,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d79aa7b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:08,639 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3af67f5f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:08,639 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:08,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38d7638f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:08,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31c3a914{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:08,643 WARN [BP-771446984-172.17.0.2-1731290595307 heartbeating to localhost/127.0.0.1:39347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:04:08,643 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:04:08,643 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:04:08,643 WARN [BP-771446984-172.17.0.2-1731290595307 heartbeating to localhost/127.0.0.1:39347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-771446984-172.17.0.2-1731290595307 (Datanode Uuid 4f58aede-1c4f-468d-ad99-38501f969cc6) service to localhost/127.0.0.1:39347 2024-11-11T02:04:08,643 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data3/current/BP-771446984-172.17.0.2-1731290595307 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:08,643 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data4/current/BP-771446984-172.17.0.2-1731290595307 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:08,644 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:04:08,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c3d0a48{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:08,647 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@784f9787{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:08,647 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:08,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@105345dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:08,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a5885a7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:08,648 WARN [BP-771446984-172.17.0.2-1731290595307 heartbeating to localhost/127.0.0.1:39347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:04:08,648 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:04:08,648 WARN [BP-771446984-172.17.0.2-1731290595307 heartbeating to localhost/127.0.0.1:39347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-771446984-172.17.0.2-1731290595307 (Datanode Uuid 8836dfa1-1616-448b-880a-a9728d93ef8d) service to localhost/127.0.0.1:39347 2024-11-11T02:04:08,648 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:04:08,649 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data1/current/BP-771446984-172.17.0.2-1731290595307 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:08,649 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/cluster_bb6d036f-4791-69f6-86de-74b14816c69e/data/data2/current/BP-771446984-172.17.0.2-1731290595307 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:08,649 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:04:08,655 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6b4d832e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:04:08,655 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@147f88bb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:08,655 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:08,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57aab5a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:08,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@372de30f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:08,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:04:08,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T02:04:08,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testCompactionRecordDoesntBlockRolling Thread=96 (was 102), OpenFileDescriptor=428 (was 437), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=32 (was 58), ProcessCount=11 (was 11), AvailableMemoryMB=6095 (was 6234) 2024-11-11T02:04:08,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRolling Thread=96, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=32, ProcessCount=11, AvailableMemoryMB=6095 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.log.dir so I do NOT create it in target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7f68fee7-3a23-7253-4b6c-981e17c60ce7/hadoop.tmp.dir so I do NOT create it in target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef, deleteOnExit=true 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/test.cache.data in system properties and HBase conf 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:04:08,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:04:08,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:04:08,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:04:08,749 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:08,753 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:08,754 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:08,754 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:08,754 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T02:04:08,755 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:08,755 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@548a1d03{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:08,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6fe19690{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:08,849 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@307e1807{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/java.io.tmpdir/jetty-localhost-36259-hadoop-hdfs-3_4_1-tests_jar-_-any-14380933212206930040/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:04:08,850 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51d868a4{HTTP/1.1, (http/1.1)}{localhost:36259} 2024-11-11T02:04:08,850 INFO [Time-limited test {}] server.Server(415): Started @145166ms 2024-11-11T02:04:08,903 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:08,906 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:08,907 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:08,907 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:08,907 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:04:08,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67ee70d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:08,908 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31864e8a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:08,980 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:08,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34b4101a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/java.io.tmpdir/jetty-localhost-41421-hadoop-hdfs-3_4_1-tests_jar-_-any-3114929417039199122/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:09,000 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@612c30eb{HTTP/1.1, (http/1.1)}{localhost:41421} 2024-11-11T02:04:09,000 INFO [Time-limited test {}] server.Server(415): Started @145316ms 2024-11-11T02:04:09,001 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:04:09,024 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:09,027 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:09,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:09,029 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:09,029 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:04:09,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73d7d88e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:09,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7aa52dc4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:09,056 WARN [Thread-1090 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data1/current/BP-585812650-172.17.0.2-1731290648715/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:09,056 WARN [Thread-1091 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data2/current/BP-585812650-172.17.0.2-1731290648715/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:09,070 WARN [Thread-1069 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:04:09,072 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb841de5833061e3a with lease ID 0x8a058aff546ff880: Processing first storage report for DS-db952a82-f11f-407c-93be-f5d7e897634b from datanode DatanodeRegistration(127.0.0.1:34805, datanodeUuid=26eb6578-7654-49d1-beae-a1fa4a4b70a9, infoPort=46739, infoSecurePort=0, ipcPort=41727, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715) 2024-11-11T02:04:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb841de5833061e3a with lease ID 0x8a058aff546ff880: from storage DS-db952a82-f11f-407c-93be-f5d7e897634b node DatanodeRegistration(127.0.0.1:34805, datanodeUuid=26eb6578-7654-49d1-beae-a1fa4a4b70a9, infoPort=46739, infoSecurePort=0, ipcPort=41727, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb841de5833061e3a with lease ID 0x8a058aff546ff880: Processing first storage report for DS-fd346be9-ab99-47b6-bd35-67cf8f5d66c9 from datanode DatanodeRegistration(127.0.0.1:34805, datanodeUuid=26eb6578-7654-49d1-beae-a1fa4a4b70a9, infoPort=46739, infoSecurePort=0, ipcPort=41727, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715) 2024-11-11T02:04:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb841de5833061e3a with lease ID 0x8a058aff546ff880: from storage DS-fd346be9-ab99-47b6-bd35-67cf8f5d66c9 node DatanodeRegistration(127.0.0.1:34805, datanodeUuid=26eb6578-7654-49d1-beae-a1fa4a4b70a9, infoPort=46739, infoSecurePort=0, ipcPort=41727, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:09,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c6855cd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/java.io.tmpdir/jetty-localhost-36321-hadoop-hdfs-3_4_1-tests_jar-_-any-10878181060788732609/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:09,127 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d903d10{HTTP/1.1, (http/1.1)}{localhost:36321} 2024-11-11T02:04:09,128 INFO [Time-limited test {}] server.Server(415): Started @145443ms 2024-11-11T02:04:09,129 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:04:09,184 WARN [Thread-1116 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data3/current/BP-585812650-172.17.0.2-1731290648715/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:09,184 WARN [Thread-1117 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data4/current/BP-585812650-172.17.0.2-1731290648715/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:09,197 WARN [Thread-1105 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:04:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca9d1bad9e5acc21 with lease ID 0x8a058aff546ff881: Processing first storage report for DS-7f8bb718-5a14-441d-8080-8f493fc67a72 from datanode DatanodeRegistration(127.0.0.1:43973, datanodeUuid=a14a035c-25fc-48e1-a1d1-2ef0a251f301, infoPort=39007, infoSecurePort=0, ipcPort=45057, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715) 2024-11-11T02:04:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca9d1bad9e5acc21 with lease ID 0x8a058aff546ff881: from storage DS-7f8bb718-5a14-441d-8080-8f493fc67a72 node DatanodeRegistration(127.0.0.1:43973, datanodeUuid=a14a035c-25fc-48e1-a1d1-2ef0a251f301, infoPort=39007, infoSecurePort=0, ipcPort=45057, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca9d1bad9e5acc21 with lease ID 0x8a058aff546ff881: Processing first storage report for DS-83a42153-0a3a-46c6-adc4-1ee4212b3e47 from datanode DatanodeRegistration(127.0.0.1:43973, datanodeUuid=a14a035c-25fc-48e1-a1d1-2ef0a251f301, infoPort=39007, infoSecurePort=0, ipcPort=45057, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715) 2024-11-11T02:04:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca9d1bad9e5acc21 with lease ID 0x8a058aff546ff881: from storage DS-83a42153-0a3a-46c6-adc4-1ee4212b3e47 node DatanodeRegistration(127.0.0.1:43973, datanodeUuid=a14a035c-25fc-48e1-a1d1-2ef0a251f301, infoPort=39007, infoSecurePort=0, ipcPort=45057, storageInfo=lv=-57;cid=testClusterID;nsid=1963610735;c=1731290648715), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T02:04:09,255 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f 2024-11-11T02:04:09,258 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/zookeeper_0, clientPort=55458, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:04:09,258 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55458 2024-11-11T02:04:09,259 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,260 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:04:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:04:09,270 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a with version=8 2024-11-11T02:04:09,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase-staging 2024-11-11T02:04:09,273 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:04:09,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:04:09,274 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38445 2024-11-11T02:04:09,276 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38445 connecting to ZooKeeper ensemble=127.0.0.1:55458 2024-11-11T02:04:09,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384450x0, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:04:09,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38445-0x1002c862f170000 connected 2024-11-11T02:04:09,292 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,294 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:09,299 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a, hbase.cluster.distributed=false 2024-11-11T02:04:09,301 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:04:09,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38445 2024-11-11T02:04:09,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38445 2024-11-11T02:04:09,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38445 2024-11-11T02:04:09,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38445 2024-11-11T02:04:09,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38445 2024-11-11T02:04:09,319 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:04:09,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:04:09,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:04:09,320 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39121 2024-11-11T02:04:09,322 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39121 connecting to ZooKeeper ensemble=127.0.0.1:55458 2024-11-11T02:04:09,322 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,324 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391210x0, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:04:09,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391210x0, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:09,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39121-0x1002c862f170001 connected 2024-11-11T02:04:09,327 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:04:09,329 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:04:09,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:04:09,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:04:09,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39121 2024-11-11T02:04:09,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39121 2024-11-11T02:04:09,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39121 2024-11-11T02:04:09,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39121 2024-11-11T02:04:09,334 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39121 2024-11-11T02:04:09,344 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:38445 2024-11-11T02:04:09,344 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:09,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:09,346 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:04:09,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,347 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:04:09,347 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,38445,1731290649272 from backup master directory 2024-11-11T02:04:09,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:09,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:09,348 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:04:09,348 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,353 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/hbase.id] with ID: 61a9eb0b-22fd-40be-a062-7f16fa3cef8b 2024-11-11T02:04:09,353 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/.tmp/hbase.id 2024-11-11T02:04:09,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:04:09,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:04:09,360 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/.tmp/hbase.id]:[hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/hbase.id] 2024-11-11T02:04:09,371 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:09,371 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:04:09,372 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-11T02:04:09,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:04:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:04:09,383 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:04:09,384 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:04:09,384 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:09,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:04:09,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:04:09,393 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store 2024-11-11T02:04:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:04:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:04:09,400 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:09,400 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:09,400 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290649400Disabling compacts and flushes for region at 1731290649400Disabling writes for close at 1731290649400Writing region close event to WAL at 1731290649400Closed at 1731290649400 2024-11-11T02:04:09,401 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/.initializing 2024-11-11T02:04:09,401 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/WALs/370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,403 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:04:09,404 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C38445%2C1731290649272, suffix=, logDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/WALs/370bc2ade342,38445,1731290649272, archiveDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/oldWALs, maxLogs=10 2024-11-11T02:04:09,420 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/WALs/370bc2ade342,38445,1731290649272/370bc2ade342%2C38445%2C1731290649272.1731290649405, exclude list is [], retry=0 2024-11-11T02:04:09,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43973,DS-7f8bb718-5a14-441d-8080-8f493fc67a72,DISK] 2024-11-11T02:04:09,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34805,DS-db952a82-f11f-407c-93be-f5d7e897634b,DISK] 2024-11-11T02:04:09,425 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/WALs/370bc2ade342,38445,1731290649272/370bc2ade342%2C38445%2C1731290649272.1731290649405 2024-11-11T02:04:09,425 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:46739:46739)] 2024-11-11T02:04:09,426 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:04:09,426 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:09,426 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,426 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:04:09,428 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:09,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:04:09,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:09,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:04:09,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:09,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:04:09,433 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:09,434 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,434 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,435 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,436 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,436 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,436 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:04:09,438 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:09,440 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:04:09,441 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739213, jitterRate=-0.06004294753074646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:04:09,441 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731290649426Initializing all the Stores at 1731290649427 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290649427Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290649427Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290649427Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290649427Cleaning up temporary data from old regions at 1731290649436 (+9 ms)Region opened successfully at 1731290649441 (+5 ms) 2024-11-11T02:04:09,442 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:04:09,445 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a2e152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:04:09,446 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:04:09,446 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:04:09,446 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:04:09,446 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:04:09,447 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T02:04:09,447 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T02:04:09,447 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:04:09,449 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:04:09,450 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:04:09,451 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:04:09,451 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:04:09,452 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:04:09,452 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:04:09,453 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:04:09,454 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:04:09,454 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:04:09,455 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:04:09,456 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:04:09,457 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:04:09,458 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:04:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,460 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,38445,1731290649272, sessionid=0x1002c862f170000, setting cluster-up flag (Was=false) 2024-11-11T02:04:09,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,464 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:04:09,465 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,470 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:04:09,471 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,38445,1731290649272 2024-11-11T02:04:09,472 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:04:09,473 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:09,474 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:04:09,474 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:04:09,474 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,38445,1731290649272 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:04:09,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:09,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:09,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:09,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:09,475 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:04:09,476 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,476 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:04:09,476 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,481 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731290679481 2024-11-11T02:04:09,481 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:04:09,481 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:04:09,482 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:04:09,482 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,483 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,483 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:04:09,483 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:04:09,483 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:04:09,483 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:04:09,483 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:04:09,483 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:04:09,485 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290649483,5,FailOnTimeoutGroup] 2024-11-11T02:04:09,485 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290649485,5,FailOnTimeoutGroup] 2024-11-11T02:04:09,485 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,485 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:04:09,485 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,485 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:04:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:04:09,492 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:04:09,492 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a 2024-11-11T02:04:09,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:04:09,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:04:09,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:09,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:04:09,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:04:09,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:09,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:04:09,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:04:09,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:09,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:04:09,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:04:09,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:09,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:04:09,512 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:04:09,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:09,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:09,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:04:09,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740 2024-11-11T02:04:09,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740 2024-11-11T02:04:09,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:04:09,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:04:09,516 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:04:09,517 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:04:09,519 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:04:09,519 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793443, jitterRate=0.00891508162021637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:04:09,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731290649503Initializing all the Stores at 1731290649504 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290649504Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290649505 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290649505Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290649505Cleaning up temporary data from old regions at 1731290649515 (+10 ms)Region opened successfully at 1731290649520 (+5 ms) 2024-11-11T02:04:09,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:04:09,520 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:04:09,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:04:09,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:04:09,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:04:09,521 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:09,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290649520Disabling compacts and flushes for region at 1731290649520Disabling writes for close at 1731290649520Writing region close event to WAL at 1731290649520Closed at 1731290649520 2024-11-11T02:04:09,522 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:09,522 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:04:09,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:04:09,523 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:04:09,524 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:04:09,536 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(746): ClusterId : 61a9eb0b-22fd-40be-a062-7f16fa3cef8b 2024-11-11T02:04:09,536 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:04:09,538 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:04:09,538 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:04:09,539 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:04:09,539 DEBUG [RS:0;370bc2ade342:39121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d066d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:04:09,550 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:39121 2024-11-11T02:04:09,550 INFO [RS:0;370bc2ade342:39121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:04:09,550 INFO [RS:0;370bc2ade342:39121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:04:09,550 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:04:09,551 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,38445,1731290649272 with port=39121, startcode=1731290649319 2024-11-11T02:04:09,551 DEBUG [RS:0;370bc2ade342:39121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:04:09,552 INFO [HMaster-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39615, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:04:09,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38445 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38445 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,554 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a 2024-11-11T02:04:09,554 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32827 2024-11-11T02:04:09,554 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:04:09,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:04:09,556 DEBUG [RS:0;370bc2ade342:39121 {}] zookeeper.ZKUtil(111): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,556 WARN [RS:0;370bc2ade342:39121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:04:09,556 INFO [RS:0;370bc2ade342:39121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:09,556 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,39121,1731290649319] 2024-11-11T02:04:09,560 INFO [RS:0;370bc2ade342:39121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:04:09,561 INFO [RS:0;370bc2ade342:39121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:04:09,562 INFO [RS:0;370bc2ade342:39121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:04:09,562 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,562 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:04:09,563 INFO [RS:0;370bc2ade342:39121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:04:09,563 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,563 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,564 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,564 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:09,564 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:04:09,564 DEBUG [RS:0;370bc2ade342:39121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:04:09,565 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,565 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,565 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,565 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,566 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,566 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,39121,1731290649319-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:04:09,580 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:04:09,581 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,39121,1731290649319-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,581 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,581 INFO [RS:0;370bc2ade342:39121 {}] regionserver.Replication(171): 370bc2ade342,39121,1731290649319 started 2024-11-11T02:04:09,594 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:09,594 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,39121,1731290649319, RpcServer on 370bc2ade342/172.17.0.2:39121, sessionid=0x1002c862f170001 2024-11-11T02:04:09,595 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:04:09,595 DEBUG [RS:0;370bc2ade342:39121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,595 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,39121,1731290649319' 2024-11-11T02:04:09,595 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:04:09,595 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,39121,1731290649319' 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:04:09,596 DEBUG [RS:0;370bc2ade342:39121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:04:09,597 INFO [RS:0;370bc2ade342:39121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:04:09,597 INFO [RS:0;370bc2ade342:39121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:04:09,674 WARN [370bc2ade342:38445 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T02:04:09,697 INFO [RS:0;370bc2ade342:39121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:04:09,699 INFO [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C39121%2C1731290649319, suffix=, logDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319, archiveDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs, maxLogs=32 2024-11-11T02:04:09,718 DEBUG [RS:0;370bc2ade342:39121 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290649700, exclude list is [], retry=0 2024-11-11T02:04:09,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34805,DS-db952a82-f11f-407c-93be-f5d7e897634b,DISK] 2024-11-11T02:04:09,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43973,DS-7f8bb718-5a14-441d-8080-8f493fc67a72,DISK] 2024-11-11T02:04:09,723 INFO [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290649700 2024-11-11T02:04:09,723 DEBUG [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46739:46739),(127.0.0.1/127.0.0.1:39007:39007)] 2024-11-11T02:04:09,925 DEBUG [370bc2ade342:38445 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:04:09,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:09,928 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,39121,1731290649319, state=OPENING 2024-11-11T02:04:09,929 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:04:09,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:09,932 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:04:09,932 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:09,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:09,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,39121,1731290649319}] 2024-11-11T02:04:09,980 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:10,087 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:04:10,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:04:10,099 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:04:10,100 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:10,100 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:04:10,101 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C39121%2C1731290649319.meta, suffix=.meta, logDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319, archiveDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs, maxLogs=32 2024-11-11T02:04:10,114 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.meta.1731290650102.meta, exclude list is [], retry=0 2024-11-11T02:04:10,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34805,DS-db952a82-f11f-407c-93be-f5d7e897634b,DISK] 2024-11-11T02:04:10,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43973,DS-7f8bb718-5a14-441d-8080-8f493fc67a72,DISK] 2024-11-11T02:04:10,118 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.meta.1731290650102.meta 2024-11-11T02:04:10,118 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:46739:46739)] 2024-11-11T02:04:10,118 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:04:10,118 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:04:10,119 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:04:10,119 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:04:10,119 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:04:10,119 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:10,119 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:04:10,119 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:04:10,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:04:10,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:04:10,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:10,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:10,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:04:10,122 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:04:10,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:10,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:10,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:04:10,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:04:10,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:10,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:10,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:04:10,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:04:10,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:10,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:10,126 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:04:10,127 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740 2024-11-11T02:04:10,128 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740 2024-11-11T02:04:10,129 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:04:10,129 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:04:10,129 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:04:10,131 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:04:10,132 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881161, jitterRate=0.12045517563819885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:04:10,132 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:04:10,132 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731290650119Writing region info on filesystem at 1731290650119Initializing all the Stores at 1731290650120 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290650120Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290650120Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290650120Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290650120Cleaning up temporary data from old regions at 1731290650129 (+9 ms)Running coprocessor post-open hooks at 1731290650132 (+3 ms)Region opened successfully at 1731290650132 2024-11-11T02:04:10,133 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731290650087 2024-11-11T02:04:10,135 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:04:10,135 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:04:10,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:10,137 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,39121,1731290649319, state=OPEN 2024-11-11T02:04:10,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:04:10,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:04:10,139 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,39121,1731290649319 2024-11-11T02:04:10,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:10,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:10,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:04:10,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,39121,1731290649319 in 207 msec 2024-11-11T02:04:10,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:04:10,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-11T02:04:10,144 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:10,144 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:04:10,145 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:04:10,145 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,39121,1731290649319, seqNum=-1] 2024-11-11T02:04:10,145 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:04:10,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:04:10,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 678 msec 2024-11-11T02:04:10,152 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731290650152, completionTime=-1 2024-11-11T02:04:10,152 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:04:10,152 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731290710154 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731290770154 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:38445, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,154 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,155 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,156 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.810sec 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:04:10,158 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:04:10,161 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:04:10,161 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:04:10,161 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,38445,1731290649272-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:10,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f21d6a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:10,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,38445,-1 for getting cluster id 2024-11-11T02:04:10,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:04:10,238 DEBUG [HMaster-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '61a9eb0b-22fd-40be-a062-7f16fa3cef8b' 2024-11-11T02:04:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:04:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "61a9eb0b-22fd-40be-a062-7f16fa3cef8b" 2024-11-11T02:04:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14287ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,38445,-1] 2024-11-11T02:04:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:04:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:10,241 INFO [HMaster-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:04:10,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60784b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:10,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:04:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,39121,1731290649319, seqNum=-1] 2024-11-11T02:04:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:04:10,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:04:10,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,38445,1731290649272 2024-11-11T02:04:10,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:10,254 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-11T02:04:10,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T02:04:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is 370bc2ade342,38445,1731290649272 2024-11-11T02:04:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b474afe 2024-11-11T02:04:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T02:04:10,258 INFO [HMaster-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T02:04:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-11T02:04:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-11T02:04:10,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:04:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-11T02:04:10,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T02:04:10,262 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:10,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-11T02:04:10,264 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T02:04:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:04:10,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741835_1011 (size=381) 2024-11-11T02:04:10,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741835_1011 (size=381) 2024-11-11T02:04:10,681 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23379243a6978d964e50a020ac1e9fed, NAME => 'TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a 2024-11-11T02:04:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741836_1012 (size=64) 2024-11-11T02:04:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741836_1012 (size=64) 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 23379243a6978d964e50a020ac1e9fed, disabling compactions & flushes 2024-11-11T02:04:10,692 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. after waiting 0 ms 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:10,692 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:10,692 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23379243a6978d964e50a020ac1e9fed: Waiting for close lock at 1731290650692Disabling compacts and flushes for region at 1731290650692Disabling writes for close at 1731290650692Writing region close event to WAL at 1731290650692Closed at 1731290650692 2024-11-11T02:04:10,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T02:04:10,694 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731290650694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290650694"}]},"ts":"1731290650694"} 2024-11-11T02:04:10,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T02:04:10,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T02:04:10,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290650697"}]},"ts":"1731290650697"} 2024-11-11T02:04:10,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-11T02:04:10,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, ASSIGN}] 2024-11-11T02:04:10,700 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, ASSIGN 2024-11-11T02:04:10,701 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, ASSIGN; state=OFFLINE, location=370bc2ade342,39121,1731290649319; forceNewPlan=false, retain=false 2024-11-11T02:04:10,853 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=23379243a6978d964e50a020ac1e9fed, regionState=OPENING, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:10,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, ASSIGN because future has completed 2024-11-11T02:04:10,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319}] 2024-11-11T02:04:10,981 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:11,021 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:11,022 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 23379243a6978d964e50a020ac1e9fed, NAME => 'TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:04:11,022 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,022 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:11,022 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,022 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,024 INFO [StoreOpener-23379243a6978d964e50a020ac1e9fed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,026 INFO [StoreOpener-23379243a6978d964e50a020ac1e9fed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23379243a6978d964e50a020ac1e9fed columnFamilyName info 2024-11-11T02:04:11,026 DEBUG [StoreOpener-23379243a6978d964e50a020ac1e9fed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:11,027 INFO [StoreOpener-23379243a6978d964e50a020ac1e9fed-1 {}] regionserver.HStore(327): Store=23379243a6978d964e50a020ac1e9fed/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:11,027 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,028 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,028 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,028 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,028 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,031 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,033 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:04:11,034 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 23379243a6978d964e50a020ac1e9fed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728539, jitterRate=-0.07361561059951782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:04:11,034 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:11,035 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 23379243a6978d964e50a020ac1e9fed: Running coprocessor pre-open hook at 1731290651023Writing region info on filesystem at 1731290651023Initializing all the Stores at 1731290651024 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290651024Cleaning up temporary data from old regions at 1731290651028 (+4 ms)Running coprocessor post-open hooks at 1731290651034 (+6 ms)Region opened successfully at 1731290651035 (+1 ms) 2024-11-11T02:04:11,036 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., pid=6, masterSystemTime=1731290651012 2024-11-11T02:04:11,038 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:11,038 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:11,039 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=23379243a6978d964e50a020ac1e9fed, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:11,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319 because future has completed 2024-11-11T02:04:11,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T02:04:11,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319 in 185 msec 2024-11-11T02:04:11,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T02:04:11,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, ASSIGN in 347 msec 2024-11-11T02:04:11,051 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T02:04:11,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731290651051"}]},"ts":"1731290651051"} 2024-11-11T02:04:11,053 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-11T02:04:11,055 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T02:04:11,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 796 msec 2024-11-11T02:04:11,983 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:12,984 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:13,139 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,140 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,140 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,140 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,141 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,141 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,144 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,144 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,144 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:04:13,666 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,666 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,671 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,671 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,671 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:13,986 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:14,987 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:15,560 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:04:15,562 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-11T02:04:15,988 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:16,989 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:17,990 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:18,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:04:18,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T02:04:18,455 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:18,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T02:04:18,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T02:04:18,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T02:04:18,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-11T02:04:18,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-11T02:04:18,992 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:19,993 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:19,995 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 after 68088ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T02:04:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38445 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T02:04:20,323 INFO [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-11T02:04:20,323 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-11T02:04:20,331 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-11T02:04:20,331 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., hostname=370bc2ade342,39121,1731290649319, seqNum=2] 2024-11-11T02:04:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:20,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:20,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/b7b450fb95c64930ab038127ba3226de is 1055, key is row0001/info:/1731290660337/Put/seqid=0 2024-11-11T02:04:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741837_1013 (size=13386) 2024-11-11T02:04:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741837_1013 (size=13386) 2024-11-11T02:04:20,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/b7b450fb95c64930ab038127ba3226de 2024-11-11T02:04:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/b7b450fb95c64930ab038127ba3226de as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de 2024-11-11T02:04:20,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de, entries=8, sequenceid=12, filesize=13.1 K 2024-11-11T02:04:20,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=18.47 KB/18918 for 23379243a6978d964e50a020ac1e9fed in 40ms, sequenceid=12, compaction requested=false 2024-11-11T02:04:20,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:20,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=19.50 KB heapSize=21.18 KB 2024-11-11T02:04:20,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a7e84c4c6f4843f28015d3003817c45d is 1055, key is row0009/info:/1731290660358/Put/seqid=0 2024-11-11T02:04:20,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741838_1014 (size=24978) 2024-11-11T02:04:20,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741838_1014 (size=24978) 2024-11-11T02:04:20,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.50 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a7e84c4c6f4843f28015d3003817c45d 2024-11-11T02:04:20,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a7e84c4c6f4843f28015d3003817c45d as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d 2024-11-11T02:04:20,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d, entries=19, sequenceid=34, filesize=24.4 K 2024-11-11T02:04:20,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.50 KB/19969, heapSize ~21.16 KB/21672, currentSize=5.13 KB/5255 for 23379243a6978d964e50a020ac1e9fed in 26ms, sequenceid=34, compaction requested=false 2024-11-11T02:04:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=37.5 K, sizeToCheck=16.0 K 2024-11-11T02:04:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:20,423 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d because midkey is the same as first or last row 2024-11-11T02:04:20,995 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:21,997 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:22,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:22,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/caa81f35354f47049843328df40ca33f is 1055, key is row0028/info:/1731290660398/Put/seqid=0 2024-11-11T02:04:22,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741839_1015 (size=13386) 2024-11-11T02:04:22,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741839_1015 (size=13386) 2024-11-11T02:04:22,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/caa81f35354f47049843328df40ca33f 2024-11-11T02:04:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/caa81f35354f47049843328df40ca33f as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f 2024-11-11T02:04:22,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f, entries=8, sequenceid=45, filesize=13.1 K 2024-11-11T02:04:22,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=12.32 KB/12612 for 23379243a6978d964e50a020ac1e9fed in 27ms, sequenceid=45, compaction requested=true 2024-11-11T02:04:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.5 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d because midkey is the same as first or last row 2024-11-11T02:04:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23379243a6978d964e50a020ac1e9fed:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:22,450 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:22,451 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 51750 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:22,451 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 23379243a6978d964e50a020ac1e9fed/info is initiating minor compaction (all files) 2024-11-11T02:04:22,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:22,451 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23379243a6978d964e50a020ac1e9fed/info in TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:22,451 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp, totalSize=50.5 K 2024-11-11T02:04:22,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=14.37 KB heapSize=15.67 KB 2024-11-11T02:04:22,452 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b450fb95c64930ab038127ba3226de, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731290660337 2024-11-11T02:04:22,452 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e84c4c6f4843f28015d3003817c45d, keycount=19, bloomtype=ROW, size=24.4 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731290660358 2024-11-11T02:04:22,453 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa81f35354f47049843328df40ca33f, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731290660398 2024-11-11T02:04:22,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/64a1079e194c4ef9a654d508fd77dbc0 is 1055, key is row0036/info:/1731290662422/Put/seqid=0 2024-11-11T02:04:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741840_1016 (size=19714) 2024-11-11T02:04:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741840_1016 (size=19714) 2024-11-11T02:04:22,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.37 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/64a1079e194c4ef9a654d508fd77dbc0 2024-11-11T02:04:22,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/64a1079e194c4ef9a654d508fd77dbc0 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0 2024-11-11T02:04:22,472 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23379243a6978d964e50a020ac1e9fed#info#compaction#35 average throughput is 8.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:22,473 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/05d9d6abce894fb9bda3d5631068ed58 is 1055, key is row0001/info:/1731290660337/Put/seqid=0 2024-11-11T02:04:22,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0, entries=14, sequenceid=62, filesize=19.3 K 2024-11-11T02:04:22,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.37 KB/14714, heapSize ~15.66 KB/16032, currentSize=11.29 KB/11561 for 23379243a6978d964e50a020ac1e9fed in 27ms, sequenceid=62, compaction requested=false 2024-11-11T02:04:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.8 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d because midkey is the same as first or last row 2024-11-11T02:04:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:22,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-11T02:04:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741841_1017 (size=41949) 2024-11-11T02:04:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741841_1017 (size=41949) 2024-11-11T02:04:22,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/2acfbd44bb1146f29ebd0782c71b6e2a is 1055, key is row0050/info:/1731290662452/Put/seqid=0 2024-11-11T02:04:22,490 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/05d9d6abce894fb9bda3d5631068ed58 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 2024-11-11T02:04:22,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741842_1018 (size=17594) 2024-11-11T02:04:22,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741842_1018 (size=17594) 2024-11-11T02:04:22,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/2acfbd44bb1146f29ebd0782c71b6e2a 2024-11-11T02:04:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/2acfbd44bb1146f29ebd0782c71b6e2a as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a 2024-11-11T02:04:22,501 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23379243a6978d964e50a020ac1e9fed/info of 23379243a6978d964e50a020ac1e9fed into 05d9d6abce894fb9bda3d5631068ed58(size=41.0 K), total size for store is 60.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:22,501 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:22,504 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., storeName=23379243a6978d964e50a020ac1e9fed/info, priority=13, startTime=1731290662449; duration=0sec 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 because midkey is the same as first or last row 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,505 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 because midkey is the same as first or last row 2024-11-11T02:04:22,506 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,506 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,506 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 because midkey is the same as first or last row 2024-11-11T02:04:22,506 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:22,506 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23379243a6978d964e50a020ac1e9fed:info 2024-11-11T02:04:22,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a, entries=12, sequenceid=77, filesize=17.2 K 2024-11-11T02:04:22,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=3.08 KB/3153 for 23379243a6978d964e50a020ac1e9fed in 28ms, sequenceid=77, compaction requested=true 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=77.4 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 because midkey is the same as first or last row 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23379243a6978d964e50a020ac1e9fed:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:22,508 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:22,509 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 79257 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:22,509 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 23379243a6978d964e50a020ac1e9fed/info is initiating minor compaction (all files) 2024-11-11T02:04:22,510 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23379243a6978d964e50a020ac1e9fed/info in TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:22,510 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp, totalSize=77.4 K 2024-11-11T02:04:22,510 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d9d6abce894fb9bda3d5631068ed58, keycount=35, bloomtype=ROW, size=41.0 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731290660337 2024-11-11T02:04:22,511 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64a1079e194c4ef9a654d508fd77dbc0, keycount=14, bloomtype=ROW, size=19.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731290662422 2024-11-11T02:04:22,511 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2acfbd44bb1146f29ebd0782c71b6e2a, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731290662452 2024-11-11T02:04:22,526 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23379243a6978d964e50a020ac1e9fed#info#compaction#37 average throughput is 15.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:22,527 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/ed0c74c6d8954fac8c887052def1f51c is 1055, key is row0001/info:/1731290660337/Put/seqid=0 2024-11-11T02:04:22,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741843_1019 (size=69472) 2024-11-11T02:04:22,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741843_1019 (size=69472) 2024-11-11T02:04:22,538 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/ed0c74c6d8954fac8c887052def1f51c as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c 2024-11-11T02:04:22,545 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23379243a6978d964e50a020ac1e9fed/info of 23379243a6978d964e50a020ac1e9fed into ed0c74c6d8954fac8c887052def1f51c(size=67.8 K), total size for store is 67.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:22,545 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., storeName=23379243a6978d964e50a020ac1e9fed/info, priority=13, startTime=1731290662508; duration=0sec 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.8 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.8 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,545 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:22,546 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.8 K, sizeToCheck=16.0 K 2024-11-11T02:04:22,546 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:22,546 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:22,546 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:22,546 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23379243a6978d964e50a020ac1e9fed:info 2024-11-11T02:04:22,998 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:24,000 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:24,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/e4fb56a179a848c0b9ea5e2912a553b5 is 1055, key is row0062/info:/1731290662481/Put/seqid=0 2024-11-11T02:04:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741844_1020 (size=13386) 2024-11-11T02:04:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741844_1020 (size=13386) 2024-11-11T02:04:24,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/e4fb56a179a848c0b9ea5e2912a553b5 2024-11-11T02:04:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/e4fb56a179a848c0b9ea5e2912a553b5 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5 2024-11-11T02:04:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5, entries=8, sequenceid=90, filesize=13.1 K 2024-11-11T02:04:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=11.29 KB/11561 for 23379243a6978d964e50a020ac1e9fed in 27ms, sequenceid=90, compaction requested=false 2024-11-11T02:04:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=80.9 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-11T02:04:24,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a901d7e7bd7d4d9cad3f9939cd1d9752 is 1055, key is row0070/info:/1731290664506/Put/seqid=0 2024-11-11T02:04:24,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741845_1021 (size=17594) 2024-11-11T02:04:24,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741845_1021 (size=17594) 2024-11-11T02:04:24,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a901d7e7bd7d4d9cad3f9939cd1d9752 2024-11-11T02:04:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/a901d7e7bd7d4d9cad3f9939cd1d9752 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752 2024-11-11T02:04:24,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752, entries=12, sequenceid=105, filesize=17.2 K 2024-11-11T02:04:24,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=10.26 KB/10510 for 23379243a6978d964e50a020ac1e9fed in 25ms, sequenceid=105, compaction requested=true 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=98.1 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23379243a6978d964e50a020ac1e9fed:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:24,557 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:24,559 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 100452 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:24,559 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 23379243a6978d964e50a020ac1e9fed/info is initiating minor compaction (all files) 2024-11-11T02:04:24,559 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23379243a6978d964e50a020ac1e9fed/info in TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:24,559 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp, totalSize=98.1 K 2024-11-11T02:04:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-11T02:04:24,560 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0c74c6d8954fac8c887052def1f51c, keycount=61, bloomtype=ROW, size=67.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731290660337 2024-11-11T02:04:24,560 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4fb56a179a848c0b9ea5e2912a553b5, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731290662481 2024-11-11T02:04:24,561 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a901d7e7bd7d4d9cad3f9939cd1d9752, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731290664506 2024-11-11T02:04:24,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/ad9ace4d32c44e32b70e6ab3f5fe079e is 1055, key is row0082/info:/1731290664534/Put/seqid=0 2024-11-11T02:04:24,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741846_1022 (size=17594) 2024-11-11T02:04:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741846_1022 (size=17594) 2024-11-11T02:04:24,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/ad9ace4d32c44e32b70e6ab3f5fe079e 2024-11-11T02:04:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/ad9ace4d32c44e32b70e6ab3f5fe079e as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ad9ace4d32c44e32b70e6ab3f5fe079e 2024-11-11T02:04:24,577 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23379243a6978d964e50a020ac1e9fed#info#compaction#41 average throughput is 20.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:24,578 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/9c2d1f7a942042d7b0daed5bec076c15 is 1055, key is row0001/info:/1731290660337/Put/seqid=0 2024-11-11T02:04:24,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741847_1023 (size=90691) 2024-11-11T02:04:24,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741847_1023 (size=90691) 2024-11-11T02:04:24,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ad9ace4d32c44e32b70e6ab3f5fe079e, entries=12, sequenceid=120, filesize=17.2 K 2024-11-11T02:04:24,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=3.08 KB/3153 for 23379243a6978d964e50a020ac1e9fed in 25ms, sequenceid=120, compaction requested=false 2024-11-11T02:04:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=115.3 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c because midkey is the same as first or last row 2024-11-11T02:04:24,590 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/9c2d1f7a942042d7b0daed5bec076c15 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15 2024-11-11T02:04:24,596 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23379243a6978d964e50a020ac1e9fed/info of 23379243a6978d964e50a020ac1e9fed into 9c2d1f7a942042d7b0daed5bec076c15(size=88.6 K), total size for store is 105.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23379243a6978d964e50a020ac1e9fed: 2024-11-11T02:04:24,596 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., storeName=23379243a6978d964e50a020ac1e9fed/info, priority=13, startTime=1731290664557; duration=0sec 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.7 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.7 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,596 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.7 K, sizeToCheck=16.0 K 2024-11-11T02:04:24,597 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-11T02:04:24,597 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:24,597 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:24,597 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23379243a6978d964e50a020ac1e9fed:info 2024-11-11T02:04:24,599 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38445 {}] assignment.AssignmentManager(1363): Split request from 370bc2ade342,39121,1731290649319, parent={ENCODED => 23379243a6978d964e50a020ac1e9fed, NAME => 'TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.', STARTKEY => '', ENDKEY => ''}, splitKey=row0064 2024-11-11T02:04:24,603 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38445 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=370bc2ade342,39121,1731290649319 2024-11-11T02:04:24,606 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38445 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=23379243a6978d964e50a020ac1e9fed, daughterA=35889e4fc68202d088bfd09b934c7648, daughterB=3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:24,607 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=23379243a6978d964e50a020ac1e9fed, daughterA=35889e4fc68202d088bfd09b934c7648, daughterB=3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:24,607 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=23379243a6978d964e50a020ac1e9fed, daughterA=35889e4fc68202d088bfd09b934c7648, daughterB=3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:24,607 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=23379243a6978d964e50a020ac1e9fed, daughterA=35889e4fc68202d088bfd09b934c7648, daughterB=3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:24,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, UNASSIGN}] 2024-11-11T02:04:24,615 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, UNASSIGN 2024-11-11T02:04:24,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=23379243a6978d964e50a020ac1e9fed, regionState=CLOSING, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:24,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, UNASSIGN because future has completed 2024-11-11T02:04:24,619 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-11T02:04:24,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319}] 2024-11-11T02:04:24,776 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,777 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-11T02:04:24,777 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 23379243a6978d964e50a020ac1e9fed, disabling compactions & flushes 2024-11-11T02:04:24,777 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:24,777 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:24,777 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. after waiting 0 ms 2024-11-11T02:04:24,777 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:24,777 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 23379243a6978d964e50a020ac1e9fed 1/1 column families, dataSize=3.08 KB heapSize=3.55 KB 2024-11-11T02:04:24,781 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/de618fdc749c41c5944b1636e31ea52d is 1055, key is row0094/info:/1731290664561/Put/seqid=0 2024-11-11T02:04:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741848_1024 (size=8118) 2024-11-11T02:04:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741848_1024 (size=8118) 2024-11-11T02:04:24,789 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.08 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/de618fdc749c41c5944b1636e31ea52d 2024-11-11T02:04:24,795 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/.tmp/info/de618fdc749c41c5944b1636e31ea52d as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/de618fdc749c41c5944b1636e31ea52d 2024-11-11T02:04:24,801 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/de618fdc749c41c5944b1636e31ea52d, entries=3, sequenceid=127, filesize=7.9 K 2024-11-11T02:04:24,802 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~3.08 KB/3153, heapSize ~3.54 KB/3624, currentSize=0 B/0 for 23379243a6978d964e50a020ac1e9fed in 25ms, sequenceid=127, compaction requested=true 2024-11-11T02:04:24,804 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752] to archive 2024-11-11T02:04:24,805 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:04:24,806 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/b7b450fb95c64930ab038127ba3226de 2024-11-11T02:04:24,807 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a7e84c4c6f4843f28015d3003817c45d 2024-11-11T02:04:24,809 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/05d9d6abce894fb9bda3d5631068ed58 2024-11-11T02:04:24,810 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/caa81f35354f47049843328df40ca33f 2024-11-11T02:04:24,811 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/64a1079e194c4ef9a654d508fd77dbc0 2024-11-11T02:04:24,812 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ed0c74c6d8954fac8c887052def1f51c 2024-11-11T02:04:24,813 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/2acfbd44bb1146f29ebd0782c71b6e2a 2024-11-11T02:04:24,814 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/e4fb56a179a848c0b9ea5e2912a553b5 2024-11-11T02:04:24,815 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/a901d7e7bd7d4d9cad3f9939cd1d9752 2024-11-11T02:04:24,821 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1 2024-11-11T02:04:24,821 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. 2024-11-11T02:04:24,821 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 23379243a6978d964e50a020ac1e9fed: Waiting for close lock at 1731290664777Running coprocessor pre-close hooks at 1731290664777Disabling compacts and flushes for region at 1731290664777Disabling writes for close at 1731290664777Obtaining lock to block concurrent updates at 1731290664777Preparing flush snapshotting stores in 23379243a6978d964e50a020ac1e9fed at 1731290664777Finished memstore snapshotting TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., syncing WAL and waiting on mvcc, flushsize=dataSize=3153, getHeapSize=3624, getOffHeapSize=0, getCellsCount=3 at 1731290664778 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. at 1731290664778Flushing 23379243a6978d964e50a020ac1e9fed/info: creating writer at 1731290664778Flushing 23379243a6978d964e50a020ac1e9fed/info: appending metadata at 1731290664781 (+3 ms)Flushing 23379243a6978d964e50a020ac1e9fed/info: closing flushed file at 1731290664781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a0b2c66: reopening flushed file at 1731290664794 (+13 ms)Finished flush of dataSize ~3.08 KB/3153, heapSize ~3.54 KB/3624, currentSize=0 B/0 for 23379243a6978d964e50a020ac1e9fed in 25ms, sequenceid=127, compaction requested=true at 1731290664803 (+9 ms)Writing region close event to WAL at 1731290664818 (+15 ms)Running coprocessor post-close hooks at 1731290664821 (+3 ms)Closed at 1731290664821 2024-11-11T02:04:24,823 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=23379243a6978d964e50a020ac1e9fed, regionState=CLOSED 2024-11-11T02:04:24,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319 because future has completed 2024-11-11T02:04:24,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-11T02:04:24,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 23379243a6978d964e50a020ac1e9fed, server=370bc2ade342,39121,1731290649319 in 207 msec 2024-11-11T02:04:24,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T02:04:24,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=23379243a6978d964e50a020ac1e9fed, UNASSIGN in 217 msec 2024-11-11T02:04:24,841 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:24,845 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=23379243a6978d964e50a020ac1e9fed, threads=3 2024-11-11T02:04:24,847 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/de618fdc749c41c5944b1636e31ea52d for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,847 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ad9ace4d32c44e32b70e6ab3f5fe079e for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,847 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15 for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,858 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/de618fdc749c41c5944b1636e31ea52d, top=true 2024-11-11T02:04:24,859 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ad9ace4d32c44e32b70e6ab3f5fe079e, top=true 2024-11-11T02:04:24,870 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d for child: 3b08ff4cb24c7fdd6c1efb73774fff72, parent: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,871 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/de618fdc749c41c5944b1636e31ea52d for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741849_1025 (size=27) 2024-11-11T02:04:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741849_1025 (size=27) 2024-11-11T02:04:24,872 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e for child: 3b08ff4cb24c7fdd6c1efb73774fff72, parent: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,872 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/ad9ace4d32c44e32b70e6ab3f5fe079e for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741850_1026 (size=27) 2024-11-11T02:04:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741850_1026 (size=27) 2024-11-11T02:04:24,882 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15 for region: 23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:24,884 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 23379243a6978d964e50a020ac1e9fed Daughter A: [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed] storefiles, Daughter B: [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d] storefiles. 2024-11-11T02:04:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741851_1027 (size=71) 2024-11-11T02:04:24,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741851_1027 (size=71) 2024-11-11T02:04:24,900 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741852_1028 (size=71) 2024-11-11T02:04:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741852_1028 (size=71) 2024-11-11T02:04:24,913 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:24,923 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-11-11T02:04:24,925 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-11-11T02:04:24,927 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731290664926"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731290664926"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731290664926"}]},"ts":"1731290664926"} 2024-11-11T02:04:24,927 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731290664926"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290664926"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731290664926"}]},"ts":"1731290664926"} 2024-11-11T02:04:24,927 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731290664926"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731290664926"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731290664926"}]},"ts":"1731290664926"} 2024-11-11T02:04:24,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=35889e4fc68202d088bfd09b934c7648, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3b08ff4cb24c7fdd6c1efb73774fff72, ASSIGN}] 2024-11-11T02:04:24,945 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3b08ff4cb24c7fdd6c1efb73774fff72, ASSIGN 2024-11-11T02:04:24,945 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=35889e4fc68202d088bfd09b934c7648, ASSIGN 2024-11-11T02:04:24,946 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3b08ff4cb24c7fdd6c1efb73774fff72, ASSIGN; state=SPLITTING_NEW, location=370bc2ade342,39121,1731290649319; forceNewPlan=false, retain=false 2024-11-11T02:04:24,946 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=35889e4fc68202d088bfd09b934c7648, ASSIGN; state=SPLITTING_NEW, location=370bc2ade342,39121,1731290649319; forceNewPlan=false, retain=false 2024-11-11T02:04:25,001 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:25,097 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3b08ff4cb24c7fdd6c1efb73774fff72, regionState=OPENING, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:25,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=35889e4fc68202d088bfd09b934c7648, regionState=OPENING, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:25,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3b08ff4cb24c7fdd6c1efb73774fff72, ASSIGN because future has completed 2024-11-11T02:04:25,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319}] 2024-11-11T02:04:25,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=35889e4fc68202d088bfd09b934c7648, ASSIGN because future has completed 2024-11-11T02:04:25,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35889e4fc68202d088bfd09b934c7648, server=370bc2ade342,39121,1731290649319}] 2024-11-11T02:04:25,265 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:25,266 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b08ff4cb24c7fdd6c1efb73774fff72, NAME => 'TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.', STARTKEY => 'row0064', ENDKEY => ''} 2024-11-11T02:04:25,267 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,267 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:25,267 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,267 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,270 INFO [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,271 INFO [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b08ff4cb24c7fdd6c1efb73774fff72 columnFamilyName info 2024-11-11T02:04:25,271 DEBUG [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:25,283 DEBUG [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-top 2024-11-11T02:04:25,288 DEBUG [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e 2024-11-11T02:04:25,293 DEBUG [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d 2024-11-11T02:04:25,293 INFO [StoreOpener-3b08ff4cb24c7fdd6c1efb73774fff72-1 {}] regionserver.HStore(327): Store=3b08ff4cb24c7fdd6c1efb73774fff72/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:25,293 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,294 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,295 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,296 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,296 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,298 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,299 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3b08ff4cb24c7fdd6c1efb73774fff72; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738636, jitterRate=-0.06077595055103302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:04:25,299 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:25,299 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3b08ff4cb24c7fdd6c1efb73774fff72: Running coprocessor pre-open hook at 1731290665268Writing region info on filesystem at 1731290665268Initializing all the Stores at 1731290665269 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290665269Cleaning up temporary data from old regions at 1731290665296 (+27 ms)Running coprocessor post-open hooks at 1731290665299 (+3 ms)Region opened successfully at 1731290665299 2024-11-11T02:04:25,300 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., pid=12, masterSystemTime=1731290665256 2024-11-11T02:04:25,300 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:25,300 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:25,300 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:25,301 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:25,302 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:25,302 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:25,302 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-top, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=113.7 K 2024-11-11T02:04:25,302 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:25,302 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:25,302 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:25,303 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 35889e4fc68202d088bfd09b934c7648, NAME => 'TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.', STARTKEY => '', ENDKEY => 'row0064'} 2024-11-11T02:04:25,303 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed, keycount=40, bloomtype=ROW, size=88.6 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731290660337 2024-11-11T02:04:25,303 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,303 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3b08ff4cb24c7fdd6c1efb73774fff72, regionState=OPEN, openSeqNum=131, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:25,303 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:25,303 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,303 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,303 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731290664534 2024-11-11T02:04:25,304 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d, keycount=3, bloomtype=ROW, size=7.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731290664561 2024-11-11T02:04:25,305 INFO [StoreOpener-35889e4fc68202d088bfd09b934c7648-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,305 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-11T02:04:25,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T02:04:25,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB 2024-11-11T02:04:25,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 because future has completed 2024-11-11T02:04:25,306 INFO [StoreOpener-35889e4fc68202d088bfd09b934c7648-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35889e4fc68202d088bfd09b934c7648 columnFamilyName info 2024-11-11T02:04:25,306 DEBUG [StoreOpener-35889e4fc68202d088bfd09b934c7648-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:25,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T02:04:25,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 in 205 msec 2024-11-11T02:04:25,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3b08ff4cb24c7fdd6c1efb73774fff72, ASSIGN in 367 msec 2024-11-11T02:04:25,320 DEBUG [StoreOpener-35889e4fc68202d088bfd09b934c7648-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-bottom 2024-11-11T02:04:25,320 INFO [StoreOpener-35889e4fc68202d088bfd09b934c7648-1 {}] regionserver.HStore(327): Store=35889e4fc68202d088bfd09b934c7648/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:25,320 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,321 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,323 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,323 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,323 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/80b87652cbb447a0bcdc69aa129cabb1 is 193, key is TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72./info:regioninfo/1731290665303/Put/seqid=0 2024-11-11T02:04:25,325 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,326 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 35889e4fc68202d088bfd09b934c7648; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718006, jitterRate=-0.08700878918170929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T02:04:25,326 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:25,326 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 35889e4fc68202d088bfd09b934c7648: Running coprocessor pre-open hook at 1731290665303Writing region info on filesystem at 1731290665303Initializing all the Stores at 1731290665305 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290665305Cleaning up temporary data from old regions at 1731290665323 (+18 ms)Running coprocessor post-open hooks at 1731290665326 (+3 ms)Region opened successfully at 1731290665326 2024-11-11T02:04:25,327 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648., pid=13, masterSystemTime=1731290665256 2024-11-11T02:04:25,327 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 35889e4fc68202d088bfd09b934c7648:info, priority=-2147483648, current under compaction store size is 2 2024-11-11T02:04:25,327 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:25,327 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-11T02:04:25,328 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:25,328 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1541): 35889e4fc68202d088bfd09b934c7648/info is initiating minor compaction (all files) 2024-11-11T02:04:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741853_1029 (size=9882) 2024-11-11T02:04:25,328 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35889e4fc68202d088bfd09b934c7648/info in TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741853_1029 (size=9882) 2024-11-11T02:04:25,328 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-bottom] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/.tmp, totalSize=88.6 K 2024-11-11T02:04:25,329 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed, keycount=40, bloomtype=ROW, size=88.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731290660337 2024-11-11T02:04:25,329 DEBUG [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:25,329 INFO [RS_OPEN_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:25,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/80b87652cbb447a0bcdc69aa129cabb1 2024-11-11T02:04:25,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=35889e4fc68202d088bfd09b934c7648, regionState=OPEN, openSeqNum=131, regionLocation=370bc2ade342,39121,1731290649319 2024-11-11T02:04:25,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35889e4fc68202d088bfd09b934c7648, server=370bc2ade342,39121,1731290649319 because future has completed 2024-11-11T02:04:25,335 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#44 average throughput is 16.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:25,336 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/9c578f6fe531408aaaddbac17894cd35 is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:25,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-11-11T02:04:25,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 35889e4fc68202d088bfd09b934c7648, server=370bc2ade342,39121,1731290649319 in 229 msec 2024-11-11T02:04:25,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-11T02:04:25,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=35889e4fc68202d088bfd09b934c7648, ASSIGN in 394 msec 2024-11-11T02:04:25,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=23379243a6978d964e50a020ac1e9fed, daughterA=35889e4fc68202d088bfd09b934c7648, daughterB=3b08ff4cb24c7fdd6c1efb73774fff72 in 738 msec 2024-11-11T02:04:25,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741854_1030 (size=40005) 2024-11-11T02:04:25,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741854_1030 (size=40005) 2024-11-11T02:04:25,350 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/9c578f6fe531408aaaddbac17894cd35 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c578f6fe531408aaaddbac17894cd35 2024-11-11T02:04:25,352 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35889e4fc68202d088bfd09b934c7648#info#compaction#45 average throughput is 15.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:25,353 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/.tmp/info/d8afc0ad01be40f99421112bc2dc867e is 1055, key is row0001/info:/1731290660337/Put/seqid=0 2024-11-11T02:04:25,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/ns/4c1ca618b6984875b5853cc5b51d82b8 is 43, key is default/ns:d/1731290650147/Put/seqid=0 2024-11-11T02:04:25,357 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into 9c578f6fe531408aaaddbac17894cd35(size=39.1 K), total size for store is 39.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:25,357 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:25,357 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290665300; duration=0sec 2024-11-11T02:04:25,357 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:25,357 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741855_1031 (size=71441) 2024-11-11T02:04:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741855_1031 (size=71441) 2024-11-11T02:04:25,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741856_1032 (size=5153) 2024-11-11T02:04:25,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741856_1032 (size=5153) 2024-11-11T02:04:25,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/ns/4c1ca618b6984875b5853cc5b51d82b8 2024-11-11T02:04:25,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/table/1032656dea2c4403952be7fd2311456e is 65, key is TestLogRolling-testLogRolling/table:state/1731290651051/Put/seqid=0 2024-11-11T02:04:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741857_1033 (size=5340) 2024-11-11T02:04:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741857_1033 (size=5340) 2024-11-11T02:04:25,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/table/1032656dea2c4403952be7fd2311456e 2024-11-11T02:04:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/80b87652cbb447a0bcdc69aa129cabb1 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/info/80b87652cbb447a0bcdc69aa129cabb1 2024-11-11T02:04:25,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/info/80b87652cbb447a0bcdc69aa129cabb1, entries=30, sequenceid=17, filesize=9.7 K 2024-11-11T02:04:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/ns/4c1ca618b6984875b5853cc5b51d82b8 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/ns/4c1ca618b6984875b5853cc5b51d82b8 2024-11-11T02:04:25,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/ns/4c1ca618b6984875b5853cc5b51d82b8, entries=2, sequenceid=17, filesize=5.0 K 2024-11-11T02:04:25,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/table/1032656dea2c4403952be7fd2311456e as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/table/1032656dea2c4403952be7fd2311456e 2024-11-11T02:04:25,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/table/1032656dea2c4403952be7fd2311456e, entries=2, sequenceid=17, filesize=5.2 K 2024-11-11T02:04:25,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 107ms, sequenceid=17, compaction requested=false 2024-11-11T02:04:25,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-11T02:04:25,796 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/.tmp/info/d8afc0ad01be40f99421112bc2dc867e as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/d8afc0ad01be40f99421112bc2dc867e 2024-11-11T02:04:25,803 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 35889e4fc68202d088bfd09b934c7648/info of 35889e4fc68202d088bfd09b934c7648 into d8afc0ad01be40f99421112bc2dc867e(size=69.8 K), total size for store is 69.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:25,803 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35889e4fc68202d088bfd09b934c7648: 2024-11-11T02:04:25,803 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648., storeName=35889e4fc68202d088bfd09b934c7648/info, priority=15, startTime=1731290665327; duration=0sec 2024-11-11T02:04:25,803 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:25,803 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35889e4fc68202d088bfd09b934c7648:info 2024-11-11T02:04:26,002 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:44696 deadline: 1731290676568, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. is not online on 370bc2ade342,39121,1731290649319 2024-11-11T02:04:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., hostname=370bc2ade342,39121,1731290649319, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., hostname=370bc2ade342,39121,1731290649319, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. is not online on 370bc2ade342,39121,1731290649319 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:04:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., hostname=370bc2ade342,39121,1731290649319, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed. is not online on 370bc2ade342,39121,1731290649319 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:04:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731290650259.23379243a6978d964e50a020ac1e9fed., hostname=370bc2ade342,39121,1731290649319, seqNum=2 from cache 2024-11-11T02:04:27,003 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:28,005 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:29,006 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:29,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,844 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,844 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,847 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,847 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,847 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:29,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,007 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:30,359 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-11T02:04:30,380 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,380 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,380 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,381 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,381 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,381 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,384 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,384 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,384 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:30,386 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-11T02:04:31,008 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:32,010 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:33,011 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:34,013 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:35,014 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:36,016 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., hostname=370bc2ade342,39121,1731290649319, seqNum=131] 2024-11-11T02:04:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:36,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:36,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/aef026d75c674f21b62db7cb6606c065 is 1055, key is row0097/info:/1731290676636/Put/seqid=0 2024-11-11T02:04:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741858_1034 (size=13394) 2024-11-11T02:04:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741858_1034 (size=13394) 2024-11-11T02:04:36,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/aef026d75c674f21b62db7cb6606c065 2024-11-11T02:04:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/aef026d75c674f21b62db7cb6606c065 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065 2024-11-11T02:04:36,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065, entries=8, sequenceid=142, filesize=13.1 K 2024-11-11T02:04:36,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=12.32 KB/12612 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 26ms, sequenceid=142, compaction requested=false 2024-11-11T02:04:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:36,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-11T02:04:36,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/df40deff0d16442fa5234a06642f5f50 is 1055, key is row0105/info:/1731290676649/Put/seqid=0 2024-11-11T02:04:36,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741859_1035 (size=18675) 2024-11-11T02:04:36,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741859_1035 (size=18675) 2024-11-11T02:04:36,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/df40deff0d16442fa5234a06642f5f50 2024-11-11T02:04:36,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/df40deff0d16442fa5234a06642f5f50 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50 2024-11-11T02:04:36,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50, entries=13, sequenceid=158, filesize=18.2 K 2024-11-11T02:04:36,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=11.29 KB/11561 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 24ms, sequenceid=158, compaction requested=true 2024-11-11T02:04:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:36,698 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:36,699 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 72074 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:36,699 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:36,699 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:36,699 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c578f6fe531408aaaddbac17894cd35, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=70.4 K 2024-11-11T02:04:36,700 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c578f6fe531408aaaddbac17894cd35, keycount=33, bloomtype=ROW, size=39.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731290662486 2024-11-11T02:04:36,700 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.Compactor(225): Compacting aef026d75c674f21b62db7cb6606c065, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731290676636 2024-11-11T02:04:36,700 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] compactions.Compactor(225): Compacting df40deff0d16442fa5234a06642f5f50, keycount=13, bloomtype=ROW, size=18.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731290676649 2024-11-11T02:04:36,711 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#50 average throughput is 27.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:36,712 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/675d276c028d4b7cb96ccca046d0b896 is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741860_1036 (size=62288) 2024-11-11T02:04:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741860_1036 (size=62288) 2024-11-11T02:04:36,723 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/675d276c028d4b7cb96ccca046d0b896 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/675d276c028d4b7cb96ccca046d0b896 2024-11-11T02:04:36,729 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into 675d276c028d4b7cb96ccca046d0b896(size=60.8 K), total size for store is 60.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:36,729 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:36,729 INFO [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290676698; duration=0sec 2024-11-11T02:04:36,729 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:36,729 DEBUG [RS:0;370bc2ade342:39121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:37,017 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:38,019 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:38,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-11T02:04:38,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/c0166cdb181c4782892fa7824b531f0d is 1055, key is row0118/info:/1731290676676/Put/seqid=0 2024-11-11T02:04:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741861_1037 (size=17606) 2024-11-11T02:04:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741861_1037 (size=17606) 2024-11-11T02:04:38,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/c0166cdb181c4782892fa7824b531f0d 2024-11-11T02:04:38,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/c0166cdb181c4782892fa7824b531f0d as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d 2024-11-11T02:04:38,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d, entries=12, sequenceid=174, filesize=17.2 K 2024-11-11T02:04:38,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=12.32 KB/12612 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 26ms, sequenceid=174, compaction requested=false 2024-11-11T02:04:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:38,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-11T02:04:38,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/0fd6421cae6e47339cff5191c893f2dc is 1055, key is row0130/info:/1731290678701/Put/seqid=0 2024-11-11T02:04:38,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741862_1038 (size=18675) 2024-11-11T02:04:38,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741862_1038 (size=18675) 2024-11-11T02:04:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T02:04:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:44696 deadline: 1731290688752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 2024-11-11T02:04:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., hostname=370bc2ade342,39121,1731290649319, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., hostname=370bc2ade342,39121,1731290649319, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:04:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., hostname=370bc2ade342,39121,1731290649319, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3b08ff4cb24c7fdd6c1efb73774fff72, server=370bc2ade342,39121,1731290649319 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T02:04:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., hostname=370bc2ade342,39121,1731290649319, seqNum=131 because the exception is null or not the one we care about 2024-11-11T02:04:39,020 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:39,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/0fd6421cae6e47339cff5191c893f2dc 2024-11-11T02:04:39,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/0fd6421cae6e47339cff5191c893f2dc as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc 2024-11-11T02:04:39,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc, entries=13, sequenceid=190, filesize=18.2 K 2024-11-11T02:04:39,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=16.42 KB/16816 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 431ms, sequenceid=190, compaction requested=true 2024-11-11T02:04:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:39,156 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:39,158 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 98569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:39,158 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:39,158 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:39,158 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/675d276c028d4b7cb96ccca046d0b896, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=96.3 K 2024-11-11T02:04:39,158 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675d276c028d4b7cb96ccca046d0b896, keycount=54, bloomtype=ROW, size=60.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731290662486 2024-11-11T02:04:39,159 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0166cdb181c4782892fa7824b531f0d, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731290676676 2024-11-11T02:04:39,159 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd6421cae6e47339cff5191c893f2dc, keycount=13, bloomtype=ROW, size=18.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731290678701 2024-11-11T02:04:39,171 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#53 average throughput is 26.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:39,172 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/b747ef1c3af34c41a363550595135ec5 is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741863_1039 (size=88792) 2024-11-11T02:04:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741863_1039 (size=88792) 2024-11-11T02:04:39,181 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/b747ef1c3af34c41a363550595135ec5 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/b747ef1c3af34c41a363550595135ec5 2024-11-11T02:04:39,188 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into b747ef1c3af34c41a363550595135ec5(size=86.7 K), total size for store is 86.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:39,188 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:39,188 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290679156; duration=0sec 2024-11-11T02:04:39,188 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:39,188 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:39,254 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T02:04:40,022 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:41,023 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:42,024 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:43,026 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:44,027 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:45,029 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:46,030 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:47,032 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:48,033 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:48,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=17.45 KB heapSize=18.98 KB 2024-11-11T02:04:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/559aac6408354f3480a481eb56cb1b6e is 1055, key is row0143/info:/1731290678726/Put/seqid=0 2024-11-11T02:04:48,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741864_1040 (size=22891) 2024-11-11T02:04:48,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741864_1040 (size=22891) 2024-11-11T02:04:48,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.45 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/559aac6408354f3480a481eb56cb1b6e 2024-11-11T02:04:48,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/559aac6408354f3480a481eb56cb1b6e as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e 2024-11-11T02:04:48,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e, entries=17, sequenceid=211, filesize=22.4 K 2024-11-11T02:04:48,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.45 KB/17867, heapSize ~18.96 KB/19416, currentSize=1.03 KB/1051 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 24ms, sequenceid=211, compaction requested=false 2024-11-11T02:04:48,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:49,034 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:50,035 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:50,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:50,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/ecaf3f5ced8d4601b017f0c816172106 is 1055, key is row0160/info:/1731290688849/Put/seqid=0 2024-11-11T02:04:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741865_1041 (size=13394) 2024-11-11T02:04:50,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741865_1041 (size=13394) 2024-11-11T02:04:50,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/ecaf3f5ced8d4601b017f0c816172106 2024-11-11T02:04:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/ecaf3f5ced8d4601b017f0c816172106 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106 2024-11-11T02:04:50,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106, entries=8, sequenceid=222, filesize=13.1 K 2024-11-11T02:04:50,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=13.34 KB/13663 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 26ms, sequenceid=222, compaction requested=true 2024-11-11T02:04:50,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:50,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:50,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:50,897 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:50,898 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 125077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:50,898 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:50,899 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:50,899 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/b747ef1c3af34c41a363550595135ec5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=122.1 K 2024-11-11T02:04:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:50,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=15.40 KB heapSize=16.77 KB 2024-11-11T02:04:50,899 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b747ef1c3af34c41a363550595135ec5, keycount=79, bloomtype=ROW, size=86.7 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731290662486 2024-11-11T02:04:50,900 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 559aac6408354f3480a481eb56cb1b6e, keycount=17, bloomtype=ROW, size=22.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731290678726 2024-11-11T02:04:50,900 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecaf3f5ced8d4601b017f0c816172106, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731290688849 2024-11-11T02:04:50,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/2491eefd92e640e9934c5b1eed2b8dd5 is 1055, key is row0168/info:/1731290690872/Put/seqid=0 2024-11-11T02:04:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741866_1042 (size=20781) 2024-11-11T02:04:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741866_1042 (size=20781) 2024-11-11T02:04:50,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.40 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/2491eefd92e640e9934c5b1eed2b8dd5 2024-11-11T02:04:50,914 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#57 average throughput is 26.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:50,915 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/0cf5098c961341359c5d2a9ae033ced3 is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/2491eefd92e640e9934c5b1eed2b8dd5 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5 2024-11-11T02:04:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741867_1043 (size=115223) 2024-11-11T02:04:50,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741867_1043 (size=115223) 2024-11-11T02:04:50,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5, entries=15, sequenceid=240, filesize=20.3 K 2024-11-11T02:04:50,925 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/0cf5098c961341359c5d2a9ae033ced3 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0cf5098c961341359c5d2a9ae033ced3 2024-11-11T02:04:50,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.40 KB/15765, heapSize ~16.76 KB/17160, currentSize=10.26 KB/10510 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 26ms, sequenceid=240, compaction requested=false 2024-11-11T02:04:50,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:50,931 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into 0cf5098c961341359c5d2a9ae033ced3(size=112.5 K), total size for store is 132.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:50,931 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:50,931 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290690897; duration=0sec 2024-11-11T02:04:50,931 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:50,931 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:51,019 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T02:04:51,019 INFO [master/370bc2ade342:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T02:04:51,037 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:52,038 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:52,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=11.29 KB heapSize=12.37 KB 2024-11-11T02:04:52,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 is 1055, key is row0183/info:/1731290690900/Put/seqid=0 2024-11-11T02:04:52,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741868_1044 (size=16553) 2024-11-11T02:04:52,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741868_1044 (size=16553) 2024-11-11T02:04:52,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.29 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 2024-11-11T02:04:52,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 2024-11-11T02:04:52,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208, entries=11, sequenceid=255, filesize=16.2 K 2024-11-11T02:04:52,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.29 KB/11561, heapSize ~12.35 KB/12648, currentSize=11.29 KB/11561 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 27ms, sequenceid=255, compaction requested=true 2024-11-11T02:04:52,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:52,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:52,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:52,951 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:52,952 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 152557 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:52,953 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:52,953 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:52,953 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0cf5098c961341359c5d2a9ae033ced3, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=149.0 K 2024-11-11T02:04:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:52,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-11T02:04:52,953 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cf5098c961341359c5d2a9ae033ced3, keycount=104, bloomtype=ROW, size=112.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731290662486 2024-11-11T02:04:52,954 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2491eefd92e640e9934c5b1eed2b8dd5, keycount=15, bloomtype=ROW, size=20.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731290690872 2024-11-11T02:04:52,954 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2f0fb7ddf84b6793b1c5b49a5cc208, keycount=11, bloomtype=ROW, size=16.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731290690900 2024-11-11T02:04:52,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/a2f69f90bf01475691f2ed5acfb659ba is 1055, key is row0194/info:/1731290692926/Put/seqid=0 2024-11-11T02:04:52,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741869_1045 (size=18688) 2024-11-11T02:04:52,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741869_1045 (size=18688) 2024-11-11T02:04:52,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/a2f69f90bf01475691f2ed5acfb659ba 2024-11-11T02:04:52,968 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#60 average throughput is 43.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:52,969 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/f9ed758855264ed59d7d30f74a04d5ff is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:52,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/a2f69f90bf01475691f2ed5acfb659ba as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba 2024-11-11T02:04:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741870_1046 (size=142907) 2024-11-11T02:04:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741870_1046 (size=142907) 2024-11-11T02:04:52,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba, entries=13, sequenceid=271, filesize=18.3 K 2024-11-11T02:04:52,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=9.24 KB/9459 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 25ms, sequenceid=271, compaction requested=false 2024-11-11T02:04:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:52,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=11.29 KB heapSize=12.37 KB 2024-11-11T02:04:52,982 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/f9ed758855264ed59d7d30f74a04d5ff as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/f9ed758855264ed59d7d30f74a04d5ff 2024-11-11T02:04:52,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/88ada2d6aee042cd94968a76b8b4501a is 1055, key is row0207/info:/1731290692954/Put/seqid=0 2024-11-11T02:04:52,989 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into f9ed758855264ed59d7d30f74a04d5ff(size=139.6 K), total size for store is 157.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:52,989 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:52,989 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290692951; duration=0sec 2024-11-11T02:04:52,989 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:52,989 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:52,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741871_1047 (size=16564) 2024-11-11T02:04:52,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741871_1047 (size=16564) 2024-11-11T02:04:53,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.29 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/88ada2d6aee042cd94968a76b8b4501a 2024-11-11T02:04:53,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/88ada2d6aee042cd94968a76b8b4501a as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a 2024-11-11T02:04:53,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a, entries=11, sequenceid=285, filesize=16.2 K 2024-11-11T02:04:53,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.29 KB/11561, heapSize ~12.35 KB/12648, currentSize=7.18 KB/7357 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 31ms, sequenceid=285, compaction requested=true 2024-11-11T02:04:53,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:53,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:53,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:53,011 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:53,012 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 178159 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:53,012 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:53,012 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:53,012 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/f9ed758855264ed59d7d30f74a04d5ff, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=174.0 K 2024-11-11T02:04:53,013 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ed758855264ed59d7d30f74a04d5ff, keycount=130, bloomtype=ROW, size=139.6 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731290662486 2024-11-11T02:04:53,013 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f69f90bf01475691f2ed5acfb659ba, keycount=13, bloomtype=ROW, size=18.3 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731290692926 2024-11-11T02:04:53,014 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ada2d6aee042cd94968a76b8b4501a, keycount=11, bloomtype=ROW, size=16.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731290692954 2024-11-11T02:04:53,024 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#62 average throughput is 51.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:53,025 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/5475d80594634c7e9c666c91512cb4d4 is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:53,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741872_1048 (size=168309) 2024-11-11T02:04:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741872_1048 (size=168309) 2024-11-11T02:04:53,033 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/5475d80594634c7e9c666c91512cb4d4 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5475d80594634c7e9c666c91512cb4d4 2024-11-11T02:04:53,039 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into 5475d80594634c7e9c666c91512cb4d4(size=164.4 K), total size for store is 164.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:53,039 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:53,038 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:53,039 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290693011; duration=0sec 2024-11-11T02:04:53,039 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:53,039 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:54,039 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:54,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-11T02:04:55,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/5b5d60fbaa2446bd9b83251663bea814 is 1055, key is row0218/info:/1731290692981/Put/seqid=0 2024-11-11T02:04:55,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741873_1049 (size=13402) 2024-11-11T02:04:55,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741873_1049 (size=13402) 2024-11-11T02:04:55,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/5b5d60fbaa2446bd9b83251663bea814 2024-11-11T02:04:55,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/5b5d60fbaa2446bd9b83251663bea814 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814 2024-11-11T02:04:55,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814, entries=8, sequenceid=298, filesize=13.1 K 2024-11-11T02:04:55,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=12.32 KB/12612 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 28ms, sequenceid=298, compaction requested=false 2024-11-11T02:04:55,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:55,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-11T02:04:55,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/cdbf5abf1d60475dabef2bfad335e129 is 1055, key is row0226/info:/1731290695003/Put/seqid=0 2024-11-11T02:04:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741874_1050 (size=18688) 2024-11-11T02:04:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741874_1050 (size=18688) 2024-11-11T02:04:55,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/cdbf5abf1d60475dabef2bfad335e129 2024-11-11T02:04:55,040 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:55,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/cdbf5abf1d60475dabef2bfad335e129 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129 2024-11-11T02:04:55,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129, entries=13, sequenceid=314, filesize=18.3 K 2024-11-11T02:04:55,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=12.32 KB/12612 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 23ms, sequenceid=314, compaction requested=true 2024-11-11T02:04:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39121 {}] regionserver.HRegion(8855): Flush requested on 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b08ff4cb24c7fdd6c1efb73774fff72:info, priority=-2147483648, current under compaction store size is 1 2024-11-11T02:04:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:55,052 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T02:04:55,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-11T02:04:55,053 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 200399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T02:04:55,053 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1541): 3b08ff4cb24c7fdd6c1efb73774fff72/info is initiating minor compaction (all files) 2024-11-11T02:04:55,053 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b08ff4cb24c7fdd6c1efb73774fff72/info in TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:55,053 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5475d80594634c7e9c666c91512cb4d4, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129] into tmpdir=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp, totalSize=195.7 K 2024-11-11T02:04:55,054 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5475d80594634c7e9c666c91512cb4d4, keycount=154, bloomtype=ROW, size=164.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731290662486 2024-11-11T02:04:55,054 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5d60fbaa2446bd9b83251663bea814, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731290692981 2024-11-11T02:04:55,055 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdbf5abf1d60475dabef2bfad335e129, keycount=13, bloomtype=ROW, size=18.3 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731290695003 2024-11-11T02:04:55,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/9daa14774b6f422bbd131a0bf66d9ec5 is 1055, key is row0239/info:/1731290695029/Put/seqid=0 2024-11-11T02:04:55,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741875_1051 (size=18688) 2024-11-11T02:04:55,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741875_1051 (size=18688) 2024-11-11T02:04:55,068 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b08ff4cb24c7fdd6c1efb73774fff72#info#compaction#66 average throughput is 58.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T02:04:55,069 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/e4fb9338b1b348c5a4ceba6b6c23eefc is 1055, key is row0064/info:/1731290662486/Put/seqid=0 2024-11-11T02:04:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741876_1052 (size=190553) 2024-11-11T02:04:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741876_1052 (size=190553) 2024-11-11T02:04:55,076 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/e4fb9338b1b348c5a4ceba6b6c23eefc as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/e4fb9338b1b348c5a4ceba6b6c23eefc 2024-11-11T02:04:55,081 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b08ff4cb24c7fdd6c1efb73774fff72/info of 3b08ff4cb24c7fdd6c1efb73774fff72 into e4fb9338b1b348c5a4ceba6b6c23eefc(size=186.1 K), total size for store is 186.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T02:04:55,082 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:55,082 INFO [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., storeName=3b08ff4cb24c7fdd6c1efb73774fff72/info, priority=13, startTime=1731290695052; duration=0sec 2024-11-11T02:04:55,082 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T02:04:55,082 DEBUG [RS:0;370bc2ade342:39121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b08ff4cb24c7fdd6c1efb73774fff72:info 2024-11-11T02:04:55,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375 2024-11-11T02:04:55,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/9daa14774b6f422bbd131a0bf66d9ec5 2024-11-11T02:04:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/9daa14774b6f422bbd131a0bf66d9ec5 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9daa14774b6f422bbd131a0bf66d9ec5 2024-11-11T02:04:55,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9daa14774b6f422bbd131a0bf66d9ec5, entries=13, sequenceid=330, filesize=18.3 K 2024-11-11T02:04:55,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=5.13 KB/5255 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 432ms, sequenceid=330, compaction requested=false 2024-11-11T02:04:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:56,041 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:57,042 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:57,063 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-11T02:04:57,082 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697063, exclude list is [], retry=0 2024-11-11T02:04:57,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43973,DS-7f8bb718-5a14-441d-8080-8f493fc67a72,DISK] 2024-11-11T02:04:57,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34805,DS-db952a82-f11f-407c-93be-f5d7e897634b,DISK] 2024-11-11T02:04:57,086 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290649700 with entries=315, filesize=303.08 KB; new WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697063 2024-11-11T02:04:57,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46739:46739),(127.0.0.1/127.0.0.1:39007:39007)] 2024-11-11T02:04:57,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290649700 is not closed yet, will try archiving it next time 2024-11-11T02:04:57,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741833_1009 (size=310363) 2024-11-11T02:04:57,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741833_1009 (size=310363) 2024-11-11T02:04:57,091 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB 2024-11-11T02:04:57,095 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/245fe0c71ce8437fbeb5d250aced4ff2 is 186, key is TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648./info:regioninfo/1731290665330/Put/seqid=0 2024-11-11T02:04:57,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741878_1054 (size=6153) 2024-11-11T02:04:57,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741878_1054 (size=6153) 2024-11-11T02:04:57,101 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/245fe0c71ce8437fbeb5d250aced4ff2 2024-11-11T02:04:57,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/.tmp/info/245fe0c71ce8437fbeb5d250aced4ff2 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/info/245fe0c71ce8437fbeb5d250aced4ff2 2024-11-11T02:04:57,111 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/info/245fe0c71ce8437fbeb5d250aced4ff2, entries=5, sequenceid=21, filesize=6.0 K 2024-11-11T02:04:57,112 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false 2024-11-11T02:04:57,112 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-11T02:04:57,112 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3b08ff4cb24c7fdd6c1efb73774fff72 1/1 column families, dataSize=5.13 KB heapSize=5.76 KB 2024-11-11T02:04:57,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/4e80c760f22e45439c8667ad49c89b98 is 1055, key is row0252/info:/1731290695053/Put/seqid=0 2024-11-11T02:04:57,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741879_1055 (size=10232) 2024-11-11T02:04:57,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741879_1055 (size=10232) 2024-11-11T02:04:57,120 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/4e80c760f22e45439c8667ad49c89b98 2024-11-11T02:04:57,126 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/.tmp/info/4e80c760f22e45439c8667ad49c89b98 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/4e80c760f22e45439c8667ad49c89b98 2024-11-11T02:04:57,131 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/4e80c760f22e45439c8667ad49c89b98, entries=5, sequenceid=339, filesize=10.0 K 2024-11-11T02:04:57,132 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.13 KB/5255, heapSize ~5.74 KB/5880, currentSize=0 B/0 for 3b08ff4cb24c7fdd6c1efb73774fff72 in 20ms, sequenceid=339, compaction requested=true 2024-11-11T02:04:57,132 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3b08ff4cb24c7fdd6c1efb73774fff72: 2024-11-11T02:04:57,132 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 35889e4fc68202d088bfd09b934c7648: 2024-11-11T02:04:57,143 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697132, exclude list is [], retry=0 2024-11-11T02:04:57,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43973,DS-7f8bb718-5a14-441d-8080-8f493fc67a72,DISK] 2024-11-11T02:04:57,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34805,DS-db952a82-f11f-407c-93be-f5d7e897634b,DISK] 2024-11-11T02:04:57,147 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697063 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697132 2024-11-11T02:04:57,147 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:46739:46739)] 2024-11-11T02:04:57,147 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697063 is not closed yet, will try archiving it next time 2024-11-11T02:04:57,147 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290649700 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs/370bc2ade342%2C39121%2C1731290649319.1731290649700 2024-11-11T02:04:57,148 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T02:04:57,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741877_1053 (size=731) 2024-11-11T02:04:57,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741877_1053 (size=731) 2024-11-11T02:04:57,149 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697063 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs/370bc2ade342%2C39121%2C1731290649319.1731290697063 2024-11-11T02:04:57,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:04:57,248 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:57,248 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:57,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:57,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:57,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:04:57,249 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:04:57,249 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1830143388, stopped=false 2024-11-11T02:04:57,249 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,38445,1731290649272 2024-11-11T02:04:57,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:57,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:57,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:57,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:57,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:04:57,252 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:57,252 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:57,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:57,252 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,39121,1731290649319' ***** 2024-11-11T02:04:57,253 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:04:57,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:57,253 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:04:57,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:57,253 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:04:57,253 INFO [RS:0;370bc2ade342:39121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(3091): Received CLOSE for 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(3091): Received CLOSE for 35889e4fc68202d088bfd09b934c7648 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,39121,1731290649319 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:04:57,254 INFO [RS:0;370bc2ade342:39121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:39121. 2024-11-11T02:04:57,255 DEBUG [RS:0;370bc2ade342:39121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:57,255 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3b08ff4cb24c7fdd6c1efb73774fff72, disabling compactions & flushes 2024-11-11T02:04:57,255 DEBUG [RS:0;370bc2ade342:39121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:57,255 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:57,255 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:57,255 INFO [RS:0;370bc2ade342:39121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:04:57,255 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. after waiting 0 ms 2024-11-11T02:04:57,255 INFO [RS:0;370bc2ade342:39121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:04:57,255 INFO [RS:0;370bc2ade342:39121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:04:57,255 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:57,255 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:04:57,256 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-11T02:04:57,256 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3b08ff4cb24c7fdd6c1efb73774fff72=TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72., 35889e4fc68202d088bfd09b934c7648=TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.} 2024-11-11T02:04:57,256 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:04:57,256 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:04:57,256 DEBUG [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35889e4fc68202d088bfd09b934c7648, 3b08ff4cb24c7fdd6c1efb73774fff72 2024-11-11T02:04:57,256 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:04:57,256 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:04:57,256 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:04:57,256 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-top, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c578f6fe531408aaaddbac17894cd35, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/675d276c028d4b7cb96ccca046d0b896, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/b747ef1c3af34c41a363550595135ec5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0cf5098c961341359c5d2a9ae033ced3, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/f9ed758855264ed59d7d30f74a04d5ff, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5475d80594634c7e9c666c91512cb4d4, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129] to archive 2024-11-11T02:04:57,258 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:04:57,260 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:57,261 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-ad9ace4d32c44e32b70e6ab3f5fe079e 2024-11-11T02:04:57,261 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-11T02:04:57,262 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:57,262 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c578f6fe531408aaaddbac17894cd35 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/9c578f6fe531408aaaddbac17894cd35 2024-11-11T02:04:57,262 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:57,262 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290697256Running coprocessor pre-close hooks at 1731290697256Disabling compacts and flushes for region at 1731290697256Disabling writes for close at 1731290697256Writing region close event to WAL at 1731290697258 (+2 ms)Running coprocessor post-close hooks at 1731290697262 (+4 ms)Closed at 1731290697262 2024-11-11T02:04:57,262 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:57,263 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/TestLogRolling-testLogRolling=23379243a6978d964e50a020ac1e9fed-de618fdc749c41c5944b1636e31ea52d 2024-11-11T02:04:57,264 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/aef026d75c674f21b62db7cb6606c065 2024-11-11T02:04:57,265 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/675d276c028d4b7cb96ccca046d0b896 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/675d276c028d4b7cb96ccca046d0b896 2024-11-11T02:04:57,266 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/df40deff0d16442fa5234a06642f5f50 2024-11-11T02:04:57,268 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/c0166cdb181c4782892fa7824b531f0d 2024-11-11T02:04:57,269 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/b747ef1c3af34c41a363550595135ec5 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/b747ef1c3af34c41a363550595135ec5 2024-11-11T02:04:57,270 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0fd6421cae6e47339cff5191c893f2dc 2024-11-11T02:04:57,272 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/559aac6408354f3480a481eb56cb1b6e 2024-11-11T02:04:57,273 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0cf5098c961341359c5d2a9ae033ced3 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/0cf5098c961341359c5d2a9ae033ced3 2024-11-11T02:04:57,274 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/ecaf3f5ced8d4601b017f0c816172106 2024-11-11T02:04:57,276 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/2491eefd92e640e9934c5b1eed2b8dd5 2024-11-11T02:04:57,277 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/f9ed758855264ed59d7d30f74a04d5ff to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/f9ed758855264ed59d7d30f74a04d5ff 2024-11-11T02:04:57,278 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/6f2f0fb7ddf84b6793b1c5b49a5cc208 2024-11-11T02:04:57,279 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/a2f69f90bf01475691f2ed5acfb659ba 2024-11-11T02:04:57,280 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5475d80594634c7e9c666c91512cb4d4 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5475d80594634c7e9c666c91512cb4d4 2024-11-11T02:04:57,281 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/88ada2d6aee042cd94968a76b8b4501a 2024-11-11T02:04:57,282 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/5b5d60fbaa2446bd9b83251663bea814 2024-11-11T02:04:57,284 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129 to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/info/cdbf5abf1d60475dabef2bfad335e129 2024-11-11T02:04:57,284 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=370bc2ade342:38445 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-11T02:04:57,285 WARN [StoreCloser-TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [9c578f6fe531408aaaddbac17894cd35=40005, aef026d75c674f21b62db7cb6606c065=13394, 675d276c028d4b7cb96ccca046d0b896=62288, df40deff0d16442fa5234a06642f5f50=18675, c0166cdb181c4782892fa7824b531f0d=17606, b747ef1c3af34c41a363550595135ec5=88792, 0fd6421cae6e47339cff5191c893f2dc=18675, 559aac6408354f3480a481eb56cb1b6e=22891, 0cf5098c961341359c5d2a9ae033ced3=115223, ecaf3f5ced8d4601b017f0c816172106=13394, 2491eefd92e640e9934c5b1eed2b8dd5=20781, f9ed758855264ed59d7d30f74a04d5ff=142907, 6f2f0fb7ddf84b6793b1c5b49a5cc208=16553, a2f69f90bf01475691f2ed5acfb659ba=18688, 5475d80594634c7e9c666c91512cb4d4=168309, 88ada2d6aee042cd94968a76b8b4501a=16564, 5b5d60fbaa2446bd9b83251663bea814=13402, cdbf5abf1d60475dabef2bfad335e129=18688] 2024-11-11T02:04:57,288 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/3b08ff4cb24c7fdd6c1efb73774fff72/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=130 2024-11-11T02:04:57,289 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3b08ff4cb24c7fdd6c1efb73774fff72: Waiting for close lock at 1731290697254Running coprocessor pre-close hooks at 1731290697254Disabling compacts and flushes for region at 1731290697254Disabling writes for close at 1731290697255 (+1 ms)Writing region close event to WAL at 1731290697285 (+30 ms)Running coprocessor post-close hooks at 1731290697289 (+4 ms)Closed at 1731290697289 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0064,1731290664603.3b08ff4cb24c7fdd6c1efb73774fff72. 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 35889e4fc68202d088bfd09b934c7648, disabling compactions & flushes 2024-11-11T02:04:57,289 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. after waiting 0 ms 2024-11-11T02:04:57,289 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:57,289 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed->hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/23379243a6978d964e50a020ac1e9fed/info/9c2d1f7a942042d7b0daed5bec076c15-bottom] to archive 2024-11-11T02:04:57,290 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T02:04:57,292 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed to hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/archive/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/info/9c2d1f7a942042d7b0daed5bec076c15.23379243a6978d964e50a020ac1e9fed 2024-11-11T02:04:57,292 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-11T02:04:57,296 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/data/default/TestLogRolling-testLogRolling/35889e4fc68202d088bfd09b934c7648/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130 2024-11-11T02:04:57,296 INFO [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:57,296 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 35889e4fc68202d088bfd09b934c7648: Waiting for close lock at 1731290697289Running coprocessor pre-close hooks at 1731290697289Disabling compacts and flushes for region at 1731290697289Disabling writes for close at 1731290697289Writing region close event to WAL at 1731290697292 (+3 ms)Running coprocessor post-close hooks at 1731290697296 (+4 ms)Closed at 1731290697296 2024-11-11T02:04:57,297 DEBUG [RS_CLOSE_REGION-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731290664603.35889e4fc68202d088bfd09b934c7648. 2024-11-11T02:04:57,456 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,39121,1731290649319; all regions closed. 2024-11-11T02:04:57,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741834_1010 (size=8107) 2024-11-11T02:04:57,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741834_1010 (size=8107) 2024-11-11T02:04:57,464 DEBUG [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs 2024-11-11T02:04:57,464 INFO [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C39121%2C1731290649319.meta:.meta(num 1731290650102) 2024-11-11T02:04:57,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741880_1056 (size=780) 2024-11-11T02:04:57,468 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/WALs/370bc2ade342,39121,1731290649319/370bc2ade342%2C39121%2C1731290649319.1731290697132 not finished, retry = 0 2024-11-11T02:04:57,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741880_1056 (size=780) 2024-11-11T02:04:57,570 INFO [regionserver/370bc2ade342:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:04:57,574 DEBUG [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/oldWALs 2024-11-11T02:04:57,574 INFO [RS:0;370bc2ade342:39121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C39121%2C1731290649319:(num 1731290697132) 2024-11-11T02:04:57,574 DEBUG [RS:0;370bc2ade342:39121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:57,574 INFO [RS:0;370bc2ade342:39121 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:04:57,574 INFO [RS:0;370bc2ade342:39121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:04:57,575 INFO [RS:0;370bc2ade342:39121 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T02:04:57,575 INFO [RS:0;370bc2ade342:39121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:04:57,575 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:04:57,576 INFO [RS:0;370bc2ade342:39121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39121 2024-11-11T02:04:57,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,39121,1731290649319 2024-11-11T02:04:57,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:04:57,579 INFO [RS:0;370bc2ade342:39121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:04:57,580 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,39121,1731290649319] 2024-11-11T02:04:57,581 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,39121,1731290649319 already deleted, retry=false 2024-11-11T02:04:57,581 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,39121,1731290649319 expired; onlineServers=0 2024-11-11T02:04:57,581 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,38445,1731290649272' ***** 2024-11-11T02:04:57,581 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:04:57,581 INFO [M:0;370bc2ade342:38445 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:04:57,581 INFO [M:0;370bc2ade342:38445 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:04:57,581 DEBUG [M:0;370bc2ade342:38445 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:04:57,582 DEBUG [M:0;370bc2ade342:38445 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:04:57,582 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:04:57,582 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290649483 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290649483,5,FailOnTimeoutGroup] 2024-11-11T02:04:57,582 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290649485 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290649485,5,FailOnTimeoutGroup] 2024-11-11T02:04:57,582 INFO [M:0;370bc2ade342:38445 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:04:57,582 INFO [M:0;370bc2ade342:38445 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:04:57,582 DEBUG [M:0;370bc2ade342:38445 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:04:57,582 INFO [M:0;370bc2ade342:38445 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:04:57,582 INFO [M:0;370bc2ade342:38445 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:04:57,582 INFO [M:0;370bc2ade342:38445 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:04:57,582 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:04:57,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:04:57,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:57,583 DEBUG [M:0;370bc2ade342:38445 {}] zookeeper.ZKUtil(347): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:04:57,583 WARN [M:0;370bc2ade342:38445 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:04:57,584 INFO [M:0;370bc2ade342:38445 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/.lastflushedseqids 2024-11-11T02:04:57,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741881_1057 (size=228) 2024-11-11T02:04:57,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741881_1057 (size=228) 2024-11-11T02:04:57,591 INFO [M:0;370bc2ade342:38445 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:04:57,591 INFO [M:0;370bc2ade342:38445 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:04:57,591 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:04:57,591 INFO [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:57,591 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:57,591 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:04:57,591 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:57,592 INFO [M:0;370bc2ade342:38445 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.36 KB 2024-11-11T02:04:57,606 DEBUG [M:0;370bc2ade342:38445 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a8c52a12774405e832dce7e96667732 is 82, key is hbase:meta,,1/info:regioninfo/1731290650136/Put/seqid=0 2024-11-11T02:04:57,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741882_1058 (size=5672) 2024-11-11T02:04:57,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741882_1058 (size=5672) 2024-11-11T02:04:57,612 INFO [M:0;370bc2ade342:38445 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a8c52a12774405e832dce7e96667732 2024-11-11T02:04:57,628 DEBUG [M:0;370bc2ade342:38445 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cf88a28864645ba8c64cb654f671790 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731290651056/Put/seqid=0 2024-11-11T02:04:57,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741883_1059 (size=7089) 2024-11-11T02:04:57,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741883_1059 (size=7089) 2024-11-11T02:04:57,633 INFO [M:0;370bc2ade342:38445 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cf88a28864645ba8c64cb654f671790 2024-11-11T02:04:57,637 INFO [M:0;370bc2ade342:38445 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cf88a28864645ba8c64cb654f671790 2024-11-11T02:04:57,651 DEBUG [M:0;370bc2ade342:38445 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb9cb1a31341e491bd71e37c1211e3 is 69, key is 370bc2ade342,39121,1731290649319/rs:state/1731290649553/Put/seqid=0 2024-11-11T02:04:57,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741884_1060 (size=5156) 2024-11-11T02:04:57,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741884_1060 (size=5156) 2024-11-11T02:04:57,656 INFO [M:0;370bc2ade342:38445 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb9cb1a31341e491bd71e37c1211e3 2024-11-11T02:04:57,673 DEBUG [M:0;370bc2ade342:38445 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef5972637eea4ab9988be1827d418393 is 52, key is load_balancer_on/state:d/1731290650253/Put/seqid=0 2024-11-11T02:04:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741885_1061 (size=5056) 2024-11-11T02:04:57,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741885_1061 (size=5056) 2024-11-11T02:04:57,678 INFO [M:0;370bc2ade342:38445 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef5972637eea4ab9988be1827d418393 2024-11-11T02:04:57,680 INFO [RS:0;370bc2ade342:39121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:04:57,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:57,680 INFO [RS:0;370bc2ade342:39121 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,39121,1731290649319; zookeeper connection closed. 2024-11-11T02:04:57,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39121-0x1002c862f170001, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:57,681 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d2930bd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d2930bd 2024-11-11T02:04:57,681 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:04:57,683 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a8c52a12774405e832dce7e96667732 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4a8c52a12774405e832dce7e96667732 2024-11-11T02:04:57,687 INFO [M:0;370bc2ade342:38445 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4a8c52a12774405e832dce7e96667732, entries=8, sequenceid=125, filesize=5.5 K 2024-11-11T02:04:57,688 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cf88a28864645ba8c64cb654f671790 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cf88a28864645ba8c64cb654f671790 2024-11-11T02:04:57,692 INFO [M:0;370bc2ade342:38445 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cf88a28864645ba8c64cb654f671790 2024-11-11T02:04:57,692 INFO [M:0;370bc2ade342:38445 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cf88a28864645ba8c64cb654f671790, entries=13, sequenceid=125, filesize=6.9 K 2024-11-11T02:04:57,693 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/20eb9cb1a31341e491bd71e37c1211e3 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/20eb9cb1a31341e491bd71e37c1211e3 2024-11-11T02:04:57,697 INFO [M:0;370bc2ade342:38445 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/20eb9cb1a31341e491bd71e37c1211e3, entries=1, sequenceid=125, filesize=5.0 K 2024-11-11T02:04:57,698 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef5972637eea4ab9988be1827d418393 as hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef5972637eea4ab9988be1827d418393 2024-11-11T02:04:57,703 INFO [M:0;370bc2ade342:38445 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32827/user/jenkins/test-data/14c14722-4858-abc7-0f19-15eed5b17a7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef5972637eea4ab9988be1827d418393, entries=1, sequenceid=125, filesize=4.9 K 2024-11-11T02:04:57,704 INFO [M:0;370bc2ade342:38445 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false 2024-11-11T02:04:57,705 INFO [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:57,705 DEBUG [M:0;370bc2ade342:38445 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290697591Disabling compacts and flushes for region at 1731290697591Disabling writes for close at 1731290697591Obtaining lock to block concurrent updates at 1731290697592 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731290697592Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1731290697592Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731290697592Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731290697592Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731290697606 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731290697606Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731290697616 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731290697628 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731290697628Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731290697637 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731290697651 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731290697651Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731290697661 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731290697672 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731290697673 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3773d868: reopening flushed file at 1731290697682 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7952c5f: reopening flushed file at 1731290697687 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5864a76: reopening flushed file at 1731290697692 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@528988f5: reopening flushed file at 1731290697698 (+6 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false at 1731290697704 (+6 ms)Writing region close event to WAL at 1731290697705 (+1 ms)Closed at 1731290697705 2024-11-11T02:04:57,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43973 is added to blk_1073741830_1006 (size=61320) 2024-11-11T02:04:57,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34805 is added to blk_1073741830_1006 (size=61320) 2024-11-11T02:04:57,707 INFO [M:0;370bc2ade342:38445 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:04:57,707 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:04:57,707 INFO [M:0;370bc2ade342:38445 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38445 2024-11-11T02:04:57,708 INFO [M:0;370bc2ade342:38445 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:04:57,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:57,809 INFO [M:0;370bc2ade342:38445 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:04:57,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38445-0x1002c862f170000, quorum=127.0.0.1:55458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:04:57,811 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c6855cd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:57,812 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d903d10{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:57,812 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:57,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7aa52dc4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:57,812 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73d7d88e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:57,813 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:04:57,813 WARN [BP-585812650-172.17.0.2-1731290648715 heartbeating to localhost/127.0.0.1:32827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:04:57,813 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:04:57,813 WARN [BP-585812650-172.17.0.2-1731290648715 heartbeating to localhost/127.0.0.1:32827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-585812650-172.17.0.2-1731290648715 (Datanode Uuid a14a035c-25fc-48e1-a1d1-2ef0a251f301) service to localhost/127.0.0.1:32827 2024-11-11T02:04:57,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data3/current/BP-585812650-172.17.0.2-1731290648715 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:57,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data4/current/BP-585812650-172.17.0.2-1731290648715 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:57,814 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:04:57,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34b4101a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:57,816 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@612c30eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:57,816 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:57,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31864e8a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:57,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67ee70d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:57,818 WARN [BP-585812650-172.17.0.2-1731290648715 heartbeating to localhost/127.0.0.1:32827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:04:57,818 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:04:57,818 WARN [BP-585812650-172.17.0.2-1731290648715 heartbeating to localhost/127.0.0.1:32827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-585812650-172.17.0.2-1731290648715 (Datanode Uuid 26eb6578-7654-49d1-beae-a1fa4a4b70a9) service to localhost/127.0.0.1:32827 2024-11-11T02:04:57,818 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:04:57,818 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data1/current/BP-585812650-172.17.0.2-1731290648715 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:57,818 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/cluster_f83c4d55-3fec-193f-7f0f-efa691e9c9ef/data/data2/current/BP-585812650-172.17.0.2-1731290648715 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:04:57,819 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:04:57,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@307e1807{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:04:57,824 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51d868a4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:04:57,824 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:04:57,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6fe19690{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:04:57,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@548a1d03{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir/,STOPPED} 2024-11-11T02:04:57,830 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:04:57,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T02:04:57,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRolling Thread=111 (was 96) - Thread LEAK? -, OpenFileDescriptor=452 (was 428) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=49 (was 32) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5926 (was 6095) 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRollOnNothingWritten Thread=111, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=49, ProcessCount=11, AvailableMemoryMB=5925 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.log.dir so I do NOT create it in target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/39be383d-0a7b-2b3e-bc12-21b7e349bb6f/hadoop.tmp.dir so I do NOT create it in target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97, deleteOnExit=true 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/test.cache.data in system properties and HBase conf 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir in system properties and HBase conf 2024-11-11T02:04:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T02:04:57,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:04:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/nfs.dump.dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/java.io.tmpdir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T02:04:57,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T02:04:57,930 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:57,934 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:57,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:57,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:57,935 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:04:57,935 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:57,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f4e4670{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:57,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4685deda{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:58,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@750e25b7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/java.io.tmpdir/jetty-localhost-46853-hadoop-hdfs-3_4_1-tests_jar-_-any-99196134813708326/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:04:58,030 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68c359c5{HTTP/1.1, (http/1.1)}{localhost:46853} 2024-11-11T02:04:58,030 INFO [Time-limited test {}] server.Server(415): Started @194346ms 2024-11-11T02:04:58,044 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:58,078 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:58,082 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:58,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:58,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:58,083 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:04:58,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e4a3730{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:58,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@353826c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:58,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c698367{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/java.io.tmpdir/jetty-localhost-32999-hadoop-hdfs-3_4_1-tests_jar-_-any-12673565240767787024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:58,180 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8c3d086{HTTP/1.1, (http/1.1)}{localhost:32999} 2024-11-11T02:04:58,180 INFO [Time-limited test {}] server.Server(415): Started @194496ms 2024-11-11T02:04:58,181 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:04:58,215 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T02:04:58,219 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T02:04:58,221 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T02:04:58,221 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T02:04:58,221 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T02:04:58,222 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b42b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,AVAILABLE} 2024-11-11T02:04:58,222 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38496a94{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T02:04:58,245 WARN [Thread-1609 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data2/current/BP-1010209951-172.17.0.2-1731290697893/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:58,245 WARN [Thread-1608 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data1/current/BP-1010209951-172.17.0.2-1731290697893/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:58,258 WARN [Thread-1587 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:04:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ac66778ed31945 with lease ID 0x4acc21bf504ea5c: Processing first storage report for DS-f4ddd07c-3951-4fbb-9092-8520981f558c from datanode DatanodeRegistration(127.0.0.1:37641, datanodeUuid=b0aee3d7-2368-4964-9650-306ff12894c1, infoPort=37017, infoSecurePort=0, ipcPort=34665, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893) 2024-11-11T02:04:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ac66778ed31945 with lease ID 0x4acc21bf504ea5c: from storage DS-f4ddd07c-3951-4fbb-9092-8520981f558c node DatanodeRegistration(127.0.0.1:37641, datanodeUuid=b0aee3d7-2368-4964-9650-306ff12894c1, infoPort=37017, infoSecurePort=0, ipcPort=34665, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ac66778ed31945 with lease ID 0x4acc21bf504ea5c: Processing first storage report for DS-e5fbeaad-349e-4719-b890-1e0503f5b3a5 from datanode DatanodeRegistration(127.0.0.1:37641, datanodeUuid=b0aee3d7-2368-4964-9650-306ff12894c1, infoPort=37017, infoSecurePort=0, ipcPort=34665, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893) 2024-11-11T02:04:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ac66778ed31945 with lease ID 0x4acc21bf504ea5c: from storage DS-e5fbeaad-349e-4719-b890-1e0503f5b3a5 node DatanodeRegistration(127.0.0.1:37641, datanodeUuid=b0aee3d7-2368-4964-9650-306ff12894c1, infoPort=37017, infoSecurePort=0, ipcPort=34665, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:58,316 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@342910ae{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/java.io.tmpdir/jetty-localhost-38291-hadoop-hdfs-3_4_1-tests_jar-_-any-16453306461880473355/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:04:58,317 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e31f159{HTTP/1.1, (http/1.1)}{localhost:38291} 2024-11-11T02:04:58,317 INFO [Time-limited test {}] server.Server(415): Started @194633ms 2024-11-11T02:04:58,318 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T02:04:58,370 WARN [Thread-1634 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data3/current/BP-1010209951-172.17.0.2-1731290697893/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:58,370 WARN [Thread-1635 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data4/current/BP-1010209951-172.17.0.2-1731290697893/current, will proceed with Du for space computation calculation, 2024-11-11T02:04:58,384 WARN [Thread-1623 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T02:04:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xccc8271da8681396 with lease ID 0x4acc21bf504ea5d: Processing first storage report for DS-aaceb091-7447-489e-9731-8cd54905d53d from datanode DatanodeRegistration(127.0.0.1:38793, datanodeUuid=6efd4ca3-7cdc-42e5-ae89-3e80740f97c3, infoPort=44355, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893) 2024-11-11T02:04:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xccc8271da8681396 with lease ID 0x4acc21bf504ea5d: from storage DS-aaceb091-7447-489e-9731-8cd54905d53d node DatanodeRegistration(127.0.0.1:38793, datanodeUuid=6efd4ca3-7cdc-42e5-ae89-3e80740f97c3, infoPort=44355, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xccc8271da8681396 with lease ID 0x4acc21bf504ea5d: Processing first storage report for DS-066681dd-f868-4268-b734-bdc9ae88af01 from datanode DatanodeRegistration(127.0.0.1:38793, datanodeUuid=6efd4ca3-7cdc-42e5-ae89-3e80740f97c3, infoPort=44355, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893) 2024-11-11T02:04:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xccc8271da8681396 with lease ID 0x4acc21bf504ea5d: from storage DS-066681dd-f868-4268-b734-bdc9ae88af01 node DatanodeRegistration(127.0.0.1:38793, datanodeUuid=6efd4ca3-7cdc-42e5-ae89-3e80740f97c3, infoPort=44355, infoSecurePort=0, ipcPort=33655, storageInfo=lv=-57;cid=testClusterID;nsid=1616236708;c=1731290697893), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T02:04:58,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c 2024-11-11T02:04:58,445 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/zookeeper_0, clientPort=55650, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T02:04:58,446 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55650 2024-11-11T02:04:58,446 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:58,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T02:04:58,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T02:04:58,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-11T02:04:58,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:04:58,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741825_1001 (size=7) 2024-11-11T02:04:58,457 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911 with version=8 2024-11-11T02:04:58,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35985/user/jenkins/test-data/ce147418-453f-9d4f-8181-f7acecf6c571/hbase-staging 2024-11-11T02:04:58,458 INFO [Time-limited test {}] client.ConnectionUtils(128): master/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:04:58,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:04:58,459 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43975 2024-11-11T02:04:58,460 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43975 connecting to ZooKeeper ensemble=127.0.0.1:55650 2024-11-11T02:04:58,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439750x0, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:04:58,464 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43975-0x1002c86ef3b0000 connected 2024-11-11T02:04:58,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,477 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,479 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:58,479 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911, hbase.cluster.distributed=false 2024-11-11T02:04:58,482 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:04:58,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43975 2024-11-11T02:04:58,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43975 2024-11-11T02:04:58,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43975 2024-11-11T02:04:58,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43975 2024-11-11T02:04:58,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43975 2024-11-11T02:04:58,501 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/370bc2ade342:0 server-side Connection retries=45 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T02:04:58,501 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T02:04:58,502 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40977 2024-11-11T02:04:58,503 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40977 connecting to ZooKeeper ensemble=127.0.0.1:55650 2024-11-11T02:04:58,504 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,505 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409770x0, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T02:04:58,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40977-0x1002c86ef3b0001 connected 2024-11-11T02:04:58,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:58,509 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T02:04:58,509 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T02:04:58,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T02:04:58,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T02:04:58,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40977 2024-11-11T02:04:58,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40977 2024-11-11T02:04:58,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40977 2024-11-11T02:04:58,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40977 2024-11-11T02:04:58,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40977 2024-11-11T02:04:58,522 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;370bc2ade342:43975 2024-11-11T02:04:58,522 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/370bc2ade342,43975,1731290698458 2024-11-11T02:04:58,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:58,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:58,524 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/370bc2ade342,43975,1731290698458 2024-11-11T02:04:58,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T02:04:58,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:58,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:58,525 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T02:04:58,525 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/370bc2ade342,43975,1731290698458 from backup master directory 2024-11-11T02:04:58,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/370bc2ade342,43975,1731290698458 2024-11-11T02:04:58,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:58,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T02:04:58,526 WARN [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:04:58,526 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=370bc2ade342,43975,1731290698458 2024-11-11T02:04:58,530 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/hbase.id] with ID: 74c10708-1483-45e8-9bc4-b90f7f378e9b 2024-11-11T02:04:58,530 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/.tmp/hbase.id 2024-11-11T02:04:58,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:04:58,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741826_1002 (size=42) 2024-11-11T02:04:58,538 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/.tmp/hbase.id]:[hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/hbase.id] 2024-11-11T02:04:58,549 INFO [master/370bc2ade342:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:58,549 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T02:04:58,550 INFO [master/370bc2ade342:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-11T02:04:58,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:58,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:04:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741827_1003 (size=196) 2024-11-11T02:04:58,562 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T02:04:58,562 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T02:04:58,563 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:58,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:04:58,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741828_1004 (size=1189) 2024-11-11T02:04:58,570 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store 2024-11-11T02:04:58,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:04:58,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741829_1005 (size=34) 2024-11-11T02:04:58,980 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:58,980 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:04:58,980 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:58,980 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:58,980 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:04:58,980 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:58,980 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:04:58,981 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290698980Disabling compacts and flushes for region at 1731290698980Disabling writes for close at 1731290698980Writing region close event to WAL at 1731290698980Closed at 1731290698980 2024-11-11T02:04:58,984 WARN [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/.initializing 2024-11-11T02:04:58,984 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/WALs/370bc2ade342,43975,1731290698458 2024-11-11T02:04:58,987 INFO [master/370bc2ade342:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:04:58,990 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C43975%2C1731290698458, suffix=, logDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/WALs/370bc2ade342,43975,1731290698458, archiveDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/oldWALs, maxLogs=10 2024-11-11T02:04:59,007 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/WALs/370bc2ade342,43975,1731290698458/370bc2ade342%2C43975%2C1731290698458.1731290698990, exclude list is [], retry=0 2024-11-11T02:04:59,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37641,DS-f4ddd07c-3951-4fbb-9092-8520981f558c,DISK] 2024-11-11T02:04:59,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38793,DS-aaceb091-7447-489e-9731-8cd54905d53d,DISK] 2024-11-11T02:04:59,011 INFO [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/WALs/370bc2ade342,43975,1731290698458/370bc2ade342%2C43975%2C1731290698458.1731290698990 2024-11-11T02:04:59,012 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37017:37017),(127.0.0.1/127.0.0.1:44355:44355)] 2024-11-11T02:04:59,012 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:04:59,012 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:59,012 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,012 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,017 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T02:04:59,017 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T02:04:59,019 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:59,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T02:04:59,021 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:59,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T02:04:59,022 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T02:04:59,023 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,023 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,024 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,025 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,025 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,026 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T02:04:59,027 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T02:04:59,028 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:04:59,029 INFO [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692511, jitterRate=-0.11942754685878754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T02:04:59,029 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731290699012Initializing all the Stores at 1731290699013 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699013Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290699015 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290699015Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290699015Cleaning up temporary data from old regions at 1731290699025 (+10 ms)Region opened successfully at 1731290699029 (+4 ms) 2024-11-11T02:04:59,030 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T02:04:59,032 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e878fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:04:59,033 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T02:04:59,033 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T02:04:59,033 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T02:04:59,033 INFO [master/370bc2ade342:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T02:04:59,034 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T02:04:59,034 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T02:04:59,034 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T02:04:59,036 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T02:04:59,037 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T02:04:59,037 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T02:04:59,037 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T02:04:59,038 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T02:04:59,039 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T02:04:59,039 INFO [master/370bc2ade342:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T02:04:59,040 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T02:04:59,041 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T02:04:59,041 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T02:04:59,042 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T02:04:59,043 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T02:04:59,044 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T02:04:59,044 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:04:59,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:59,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:59,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,046 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=370bc2ade342,43975,1731290698458, sessionid=0x1002c86ef3b0000, setting cluster-up flag (Was=false) 2024-11-11T02:04:59,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,050 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T02:04:59,051 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,43975,1731290698458 2024-11-11T02:04:59,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,056 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T02:04:59,057 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=370bc2ade342,43975,1731290698458 2024-11-11T02:04:59,058 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T02:04:59,060 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:59,060 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T02:04:59,060 INFO [master/370bc2ade342:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T02:04:59,061 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 370bc2ade342,43975,1731290698458 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T02:04:59,062 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:59,062 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:59,062 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:59,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/370bc2ade342:0, corePoolSize=5, maxPoolSize=5 2024-11-11T02:04:59,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/370bc2ade342:0, corePoolSize=10, maxPoolSize=10 2024-11-11T02:04:59,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:04:59,063 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731290729064 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T02:04:59,064 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T02:04:59,064 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:59,065 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T02:04:59,065 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,065 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T02:04:59,065 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T02:04:59,065 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T02:04:59,066 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,066 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T02:04:59,066 INFO [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T02:04:59,066 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T02:04:59,067 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290699066,5,FailOnTimeoutGroup] 2024-11-11T02:04:59,069 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290699067,5,FailOnTimeoutGroup] 2024-11-11T02:04:59,069 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,069 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T02:04:59,069 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,069 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:04:59,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741831_1007 (size=1321) 2024-11-11T02:04:59,075 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T02:04:59,075 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911 2024-11-11T02:04:59,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:04:59,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741832_1008 (size=32) 2024-11-11T02:04:59,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:59,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:04:59,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:04:59,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:04:59,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:04:59,087 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:04:59,088 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:04:59,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:04:59,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:04:59,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:04:59,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740 2024-11-11T02:04:59,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740 2024-11-11T02:04:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:04:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:04:59,092 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:04:59,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:04:59,095 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T02:04:59,095 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755661, jitterRate=-0.039127618074417114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731290699084Initializing all the Stores at 1731290699085 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699085Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699085Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290699085Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699085Cleaning up temporary data from old regions at 1731290699092 (+7 ms)Region opened successfully at 1731290699095 (+3 ms) 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:04:59,096 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:04:59,096 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:59,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290699096Disabling compacts and flushes for region at 1731290699096Disabling writes for close at 1731290699096Writing region close event to WAL at 1731290699096Closed at 1731290699096 2024-11-11T02:04:59,097 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:59,097 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T02:04:59,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T02:04:59,098 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:04:59,099 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T02:04:59,114 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(746): ClusterId : 74c10708-1483-45e8-9bc4-b90f7f378e9b 2024-11-11T02:04:59,115 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T02:04:59,117 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T02:04:59,117 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T02:04:59,119 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T02:04:59,120 DEBUG [RS:0;370bc2ade342:40977 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1641f923, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=370bc2ade342/172.17.0.2:0 2024-11-11T02:04:59,134 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;370bc2ade342:40977 2024-11-11T02:04:59,134 INFO [RS:0;370bc2ade342:40977 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T02:04:59,134 INFO [RS:0;370bc2ade342:40977 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T02:04:59,134 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T02:04:59,135 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(2659): reportForDuty to master=370bc2ade342,43975,1731290698458 with port=40977, startcode=1731290698501 2024-11-11T02:04:59,135 DEBUG [RS:0;370bc2ade342:40977 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T02:04:59,136 INFO [HMaster-EventLoopGroup-11-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T02:04:59,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43975 {}] master.ServerManager(517): Registering regionserver=370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,138 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911 2024-11-11T02:04:59,138 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44375 2024-11-11T02:04:59,138 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T02:04:59,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:04:59,140 DEBUG [RS:0;370bc2ade342:40977 {}] zookeeper.ZKUtil(111): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,140 WARN [RS:0;370bc2ade342:40977 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T02:04:59,140 INFO [RS:0;370bc2ade342:40977 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:59,140 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,140 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [370bc2ade342,40977,1731290698501] 2024-11-11T02:04:59,143 INFO [RS:0;370bc2ade342:40977 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T02:04:59,144 INFO [RS:0;370bc2ade342:40977 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T02:04:59,145 INFO [RS:0;370bc2ade342:40977 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T02:04:59,145 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,145 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T02:04:59,146 INFO [RS:0;370bc2ade342:40977 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T02:04:59,146 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/370bc2ade342:0, corePoolSize=2, maxPoolSize=2 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/370bc2ade342:0, corePoolSize=1, maxPoolSize=1 2024-11-11T02:04:59,146 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:04:59,147 DEBUG [RS:0;370bc2ade342:40977 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/370bc2ade342:0, corePoolSize=3, maxPoolSize=3 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,147 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40977,1731290698501-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:04:59,163 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T02:04:59,163 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,40977,1731290698501-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,163 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,163 INFO [RS:0;370bc2ade342:40977 {}] regionserver.Replication(171): 370bc2ade342,40977,1731290698501 started 2024-11-11T02:04:59,175 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,176 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1482): Serving as 370bc2ade342,40977,1731290698501, RpcServer on 370bc2ade342/172.17.0.2:40977, sessionid=0x1002c86ef3b0001 2024-11-11T02:04:59,176 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T02:04:59,176 DEBUG [RS:0;370bc2ade342:40977 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,176 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,40977,1731290698501' 2024-11-11T02:04:59,176 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '370bc2ade342,40977,1731290698501' 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T02:04:59,177 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T02:04:59,178 DEBUG [RS:0;370bc2ade342:40977 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T02:04:59,178 INFO [RS:0;370bc2ade342:40977 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T02:04:59,178 INFO [RS:0;370bc2ade342:40977 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T02:04:59,249 WARN [370bc2ade342:43975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T02:04:59,278 INFO [RS:0;370bc2ade342:40977 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:04:59,280 INFO [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C40977%2C1731290698501, suffix=, logDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501, archiveDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs, maxLogs=32 2024-11-11T02:04:59,292 DEBUG [RS:0;370bc2ade342:40977 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501/370bc2ade342%2C40977%2C1731290698501.1731290699280, exclude list is [], retry=0 2024-11-11T02:04:59,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37641,DS-f4ddd07c-3951-4fbb-9092-8520981f558c,DISK] 2024-11-11T02:04:59,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38793,DS-aaceb091-7447-489e-9731-8cd54905d53d,DISK] 2024-11-11T02:04:59,296 INFO [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501/370bc2ade342%2C40977%2C1731290698501.1731290699280 2024-11-11T02:04:59,297 DEBUG [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37017:37017),(127.0.0.1/127.0.0.1:44355:44355)] 2024-11-11T02:04:59,499 DEBUG [370bc2ade342:43975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T02:04:59,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,503 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,40977,1731290698501, state=OPENING 2024-11-11T02:04:59,505 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T02:04:59,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,509 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T02:04:59,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,40977,1731290698501}] 2024-11-11T02:04:59,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:59,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:59,664 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T02:04:59,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T02:04:59,673 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T02:04:59,673 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:59,674 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T02:04:59,677 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=370bc2ade342%2C40977%2C1731290698501.meta, suffix=.meta, logDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501, archiveDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs, maxLogs=32 2024-11-11T02:04:59,692 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501/370bc2ade342%2C40977%2C1731290698501.meta.1731290699677.meta, exclude list is [], retry=0 2024-11-11T02:04:59,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38793,DS-aaceb091-7447-489e-9731-8cd54905d53d,DISK] 2024-11-11T02:04:59,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37641,DS-f4ddd07c-3951-4fbb-9092-8520981f558c,DISK] 2024-11-11T02:04:59,696 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/370bc2ade342,40977,1731290698501/370bc2ade342%2C40977%2C1731290698501.meta.1731290699677.meta 2024-11-11T02:04:59,696 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37017:37017),(127.0.0.1/127.0.0.1:44355:44355)] 2024-11-11T02:04:59,696 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T02:04:59,696 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T02:04:59,696 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T02:04:59,696 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T02:04:59,697 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T02:04:59,697 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T02:04:59,697 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T02:04:59,697 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T02:04:59,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T02:04:59,699 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T02:04:59,699 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T02:04:59,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T02:04:59,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T02:04:59,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T02:04:59,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T02:04:59,702 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T02:04:59,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T02:04:59,703 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T02:04:59,703 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T02:04:59,703 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740 2024-11-11T02:04:59,704 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740 2024-11-11T02:04:59,706 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T02:04:59,706 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T02:04:59,706 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T02:04:59,707 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T02:04:59,708 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=830949, jitterRate=0.056606993079185486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T02:04:59,708 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T02:04:59,709 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731290699697Writing region info on filesystem at 1731290699697Initializing all the Stores at 1731290699698 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699698Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699698Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731290699698Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731290699698Cleaning up temporary data from old regions at 1731290699706 (+8 ms)Running coprocessor post-open hooks at 1731290699708 (+2 ms)Region opened successfully at 1731290699709 (+1 ms) 2024-11-11T02:04:59,710 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731290699663 2024-11-11T02:04:59,711 DEBUG [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T02:04:59,711 INFO [RS_OPEN_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T02:04:59,712 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,712 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 370bc2ade342,40977,1731290698501, state=OPEN 2024-11-11T02:04:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:04:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T02:04:59,714 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:59,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T02:04:59,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T02:04:59,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=370bc2ade342,40977,1731290698501 in 205 msec 2024-11-11T02:04:59,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T02:04:59,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-11T02:04:59,718 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T02:04:59,718 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T02:04:59,719 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:04:59,719 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,40977,1731290698501, seqNum=-1] 2024-11-11T02:04:59,720 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:04:59,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:04:59,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 664 msec 2024-11-11T02:04:59,724 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731290699724, completionTime=-1 2024-11-11T02:04:59,724 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T02:04:59,725 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T02:04:59,726 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T02:04:59,726 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731290759726 2024-11-11T02:04:59,726 INFO [master/370bc2ade342:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731290819726 2024-11-11T02:04:59,726 INFO [master/370bc2ade342:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-11T02:04:59,726 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,727 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,727 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,727 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-370bc2ade342:43975, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,727 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,727 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,728 DEBUG [master/370bc2ade342:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.204sec 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T02:04:59,730 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T02:04:59,732 DEBUG [master/370bc2ade342:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T02:04:59,732 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T02:04:59,732 INFO [master/370bc2ade342:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=370bc2ade342,43975,1731290698458-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T02:04:59,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67315410, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:59,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 370bc2ade342,43975,-1 for getting cluster id 2024-11-11T02:04:59,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T02:04:59,816 DEBUG [HMaster-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74c10708-1483-45e8-9bc4-b90f7f378e9b' 2024-11-11T02:04:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T02:04:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74c10708-1483-45e8-9bc4-b90f7f378e9b" 2024-11-11T02:04:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be6e344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [370bc2ade342,43975,-1] 2024-11-11T02:04:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T02:04:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:59,818 INFO [HMaster-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T02:04:59,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7f3295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T02:04:59,819 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T02:04:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=370bc2ade342,40977,1731290698501, seqNum=-1] 2024-11-11T02:04:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T02:04:59,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T02:04:59,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=370bc2ade342,43975,1731290698458 2024-11-11T02:04:59,823 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T02:04:59,825 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-11T02:04:59,825 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T02:04:59,826 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T02:04:59,828 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs, maxLogs=32 2024-11-11T02:04:59,838 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699828, exclude list is [], retry=0 2024-11-11T02:04:59,841 DEBUG [AsyncFSWAL-13-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37641,DS-f4ddd07c-3951-4fbb-9092-8520981f558c,DISK] 2024-11-11T02:04:59,841 DEBUG [AsyncFSWAL-13-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38793,DS-aaceb091-7447-489e-9731-8cd54905d53d,DISK] 2024-11-11T02:04:59,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699828 2024-11-11T02:04:59,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37017:37017),(127.0.0.1/127.0.0.1:44355:44355)] 2024-11-11T02:04:59,855 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699843, exclude list is [], retry=0 2024-11-11T02:04:59,857 DEBUG [AsyncFSWAL-13-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38793,DS-aaceb091-7447-489e-9731-8cd54905d53d,DISK] 2024-11-11T02:04:59,858 DEBUG [AsyncFSWAL-13-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37641,DS-f4ddd07c-3951-4fbb-9092-8520981f558c,DISK] 2024-11-11T02:04:59,859 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699828 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699843 2024-11-11T02:04:59,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44355:44355),(127.0.0.1/127.0.0.1:37017:37017)] 2024-11-11T02:04:59,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699828 is not closed yet, will try archiving it next time 2024-11-11T02:04:59,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741835_1011 (size=93) 2024-11-11T02:04:59,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741836_1012 (size=93) 2024-11-11T02:04:59,862 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/WALs/test.com,8080,1/test.com%2C8080%2C1.1731290699828 to hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs/test.com%2C8080%2C1.1731290699828 2024-11-11T02:04:59,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741835_1011 (size=93) 2024-11-11T02:04:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741836_1012 (size=93) 2024-11-11T02:04:59,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs 2024-11-11T02:04:59,865 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL test.com%2C8080%2C1:(num 1731290699843) 2024-11-11T02:04:59,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T02:04:59,865 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:59,865 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:59,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:59,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:59,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T02:04:59,866 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T02:04:59,866 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=781907132, stopped=false 2024-11-11T02:04:59,866 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=370bc2ade342,43975,1731290698458 2024-11-11T02:04:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T02:04:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:04:59,867 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:04:59,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T02:04:59,867 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:59,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:59,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:59,867 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '370bc2ade342,40977,1731290698501' ***** 2024-11-11T02:04:59,867 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T02:04:59,868 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T02:04:59,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(959): stopping server 370bc2ade342,40977,1731290698501 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;370bc2ade342:40977. 2024-11-11T02:04:59,868 DEBUG [RS:0;370bc2ade342:40977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T02:04:59,868 DEBUG [RS:0;370bc2ade342:40977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T02:04:59,868 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T02:04:59,869 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T02:04:59,869 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T02:04:59,869 DEBUG [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T02:04:59,869 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T02:04:59,869 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T02:04:59,869 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T02:04:59,869 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T02:04:59,869 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T02:04:59,869 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-11T02:04:59,884 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/.tmp/ns/d1106bd540704058824ad481d514f771 is 43, key is default/ns:d/1731290699721/Put/seqid=0 2024-11-11T02:04:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741837_1013 (size=5153) 2024-11-11T02:04:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741837_1013 (size=5153) 2024-11-11T02:04:59,889 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/.tmp/ns/d1106bd540704058824ad481d514f771 2024-11-11T02:04:59,894 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/.tmp/ns/d1106bd540704058824ad481d514f771 as hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/ns/d1106bd540704058824ad481d514f771 2024-11-11T02:04:59,899 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/ns/d1106bd540704058824ad481d514f771, entries=2, sequenceid=6, filesize=5.0 K 2024-11-11T02:04:59,900 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false 2024-11-11T02:04:59,900 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T02:04:59,905 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T02:04:59,905 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T02:04:59,905 INFO [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T02:04:59,905 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731290699869Running coprocessor pre-close hooks at 1731290699869Disabling compacts and flushes for region at 1731290699869Disabling writes for close at 1731290699869Obtaining lock to block concurrent updates at 1731290699869Preparing flush snapshotting stores in 1588230740 at 1731290699869Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731290699870 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731290699870Flushing 1588230740/ns: creating writer at 1731290699870Flushing 1588230740/ns: appending metadata at 1731290699883 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731290699883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e48d10e: reopening flushed file at 1731290699894 (+11 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1731290699900 (+6 ms)Writing region close event to WAL at 1731290699901 (+1 ms)Running coprocessor post-close hooks at 1731290699905 (+4 ms)Closed at 1731290699905 2024-11-11T02:04:59,905 DEBUG [RS_CLOSE_META-regionserver/370bc2ade342:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T02:05:00,045 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42151/user/jenkins/test-data/88351776-203a-4cbd-3f8b-501d3bf03bf1/WALs/370bc2ade342,33569,1731290570006/370bc2ade342%2C33569%2C1731290570006.1731290570426 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor185.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-11T02:05:00,069 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(976): stopping server 370bc2ade342,40977,1731290698501; all regions closed. 2024-11-11T02:05:00,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741834_1010 (size=1152) 2024-11-11T02:05:00,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741834_1010 (size=1152) 2024-11-11T02:05:00,078 DEBUG [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs 2024-11-11T02:05:00,078 INFO [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C40977%2C1731290698501.meta:.meta(num 1731290699677) 2024-11-11T02:05:00,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741833_1009 (size=93) 2024-11-11T02:05:00,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741833_1009 (size=93) 2024-11-11T02:05:00,085 DEBUG [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/oldWALs 2024-11-11T02:05:00,085 INFO [RS:0;370bc2ade342:40977 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 370bc2ade342%2C40977%2C1731290698501:(num 1731290699280) 2024-11-11T02:05:00,085 DEBUG [RS:0;370bc2ade342:40977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T02:05:00,085 INFO [RS:0;370bc2ade342:40977 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T02:05:00,085 INFO [RS:0;370bc2ade342:40977 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:05:00,086 INFO [RS:0;370bc2ade342:40977 {}] hbase.ChoreService(370): Chore service for: regionserver/370bc2ade342:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T02:05:00,086 INFO [RS:0;370bc2ade342:40977 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:05:00,086 INFO [regionserver/370bc2ade342:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:05:00,086 INFO [RS:0;370bc2ade342:40977 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40977 2024-11-11T02:05:00,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/370bc2ade342,40977,1731290698501 2024-11-11T02:05:00,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T02:05:00,088 INFO [RS:0;370bc2ade342:40977 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:05:00,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [370bc2ade342,40977,1731290698501] 2024-11-11T02:05:00,090 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/370bc2ade342,40977,1731290698501 already deleted, retry=false 2024-11-11T02:05:00,090 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 370bc2ade342,40977,1731290698501 expired; onlineServers=0 2024-11-11T02:05:00,090 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '370bc2ade342,43975,1731290698458' ***** 2024-11-11T02:05:00,090 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T02:05:00,090 INFO [M:0;370bc2ade342:43975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T02:05:00,090 INFO [M:0;370bc2ade342:43975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T02:05:00,090 DEBUG [M:0;370bc2ade342:43975 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T02:05:00,090 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T02:05:00,090 DEBUG [M:0;370bc2ade342:43975 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T02:05:00,090 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290699066 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.large.0-1731290699066,5,FailOnTimeoutGroup] 2024-11-11T02:05:00,090 DEBUG [master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290699067 {}] cleaner.HFileCleaner(306): Exit Thread[master/370bc2ade342:0:becomeActiveMaster-HFileCleaner.small.0-1731290699067,5,FailOnTimeoutGroup] 2024-11-11T02:05:00,091 INFO [M:0;370bc2ade342:43975 {}] hbase.ChoreService(370): Chore service for: master/370bc2ade342:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T02:05:00,091 INFO [M:0;370bc2ade342:43975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T02:05:00,091 DEBUG [M:0;370bc2ade342:43975 {}] master.HMaster(1795): Stopping service threads 2024-11-11T02:05:00,091 INFO [M:0;370bc2ade342:43975 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T02:05:00,091 INFO [M:0;370bc2ade342:43975 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T02:05:00,091 INFO [M:0;370bc2ade342:43975 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T02:05:00,091 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T02:05:00,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T02:05:00,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T02:05:00,092 DEBUG [M:0;370bc2ade342:43975 {}] zookeeper.ZKUtil(347): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T02:05:00,092 WARN [M:0;370bc2ade342:43975 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T02:05:00,093 INFO [M:0;370bc2ade342:43975 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/.lastflushedseqids 2024-11-11T02:05:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741838_1014 (size=99) 2024-11-11T02:05:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741838_1014 (size=99) 2024-11-11T02:05:00,099 INFO [M:0;370bc2ade342:43975 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T02:05:00,099 INFO [M:0;370bc2ade342:43975 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T02:05:00,099 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T02:05:00,099 INFO [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:05:00,099 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:05:00,099 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T02:05:00,099 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:05:00,099 INFO [M:0;370bc2ade342:43975 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-11T02:05:00,117 DEBUG [M:0;370bc2ade342:43975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aca0ab7fc1224482a7284576fc9400a9 is 82, key is hbase:meta,,1/info:regioninfo/1731290699712/Put/seqid=0 2024-11-11T02:05:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741839_1015 (size=5672) 2024-11-11T02:05:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741839_1015 (size=5672) 2024-11-11T02:05:00,122 INFO [M:0;370bc2ade342:43975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aca0ab7fc1224482a7284576fc9400a9 2024-11-11T02:05:00,140 DEBUG [M:0;370bc2ade342:43975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ec9cfd028b24c54b916cbf238525237 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731290699724/Put/seqid=0 2024-11-11T02:05:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741840_1016 (size=5275) 2024-11-11T02:05:00,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741840_1016 (size=5275) 2024-11-11T02:05:00,145 INFO [M:0;370bc2ade342:43975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ec9cfd028b24c54b916cbf238525237 2024-11-11T02:05:00,163 DEBUG [M:0;370bc2ade342:43975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44e82b0de72a486387bd26c585cb0169 is 69, key is 370bc2ade342,40977,1731290698501/rs:state/1731290699137/Put/seqid=0 2024-11-11T02:05:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741841_1017 (size=5156) 2024-11-11T02:05:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741841_1017 (size=5156) 2024-11-11T02:05:00,169 INFO [M:0;370bc2ade342:43975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44e82b0de72a486387bd26c585cb0169 2024-11-11T02:05:00,186 DEBUG [M:0;370bc2ade342:43975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/225fba670a48464dbda22c3baab5b05c is 52, key is load_balancer_on/state:d/1731290699824/Put/seqid=0 2024-11-11T02:05:00,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:05:00,189 INFO [RS:0;370bc2ade342:40977 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:05:00,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40977-0x1002c86ef3b0001, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:05:00,189 INFO [RS:0;370bc2ade342:40977 {}] regionserver.HRegionServer(1031): Exiting; stopping=370bc2ade342,40977,1731290698501; zookeeper connection closed. 2024-11-11T02:05:00,189 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fd53a58 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fd53a58 2024-11-11T02:05:00,190 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T02:05:00,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741842_1018 (size=5056) 2024-11-11T02:05:00,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741842_1018 (size=5056) 2024-11-11T02:05:00,192 INFO [M:0;370bc2ade342:43975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/225fba670a48464dbda22c3baab5b05c 2024-11-11T02:05:00,197 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aca0ab7fc1224482a7284576fc9400a9 as hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aca0ab7fc1224482a7284576fc9400a9 2024-11-11T02:05:00,201 INFO [M:0;370bc2ade342:43975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aca0ab7fc1224482a7284576fc9400a9, entries=8, sequenceid=29, filesize=5.5 K 2024-11-11T02:05:00,202 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ec9cfd028b24c54b916cbf238525237 as hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ec9cfd028b24c54b916cbf238525237 2024-11-11T02:05:00,206 INFO [M:0;370bc2ade342:43975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ec9cfd028b24c54b916cbf238525237, entries=3, sequenceid=29, filesize=5.2 K 2024-11-11T02:05:00,207 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/44e82b0de72a486387bd26c585cb0169 as hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44e82b0de72a486387bd26c585cb0169 2024-11-11T02:05:00,212 INFO [M:0;370bc2ade342:43975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/44e82b0de72a486387bd26c585cb0169, entries=1, sequenceid=29, filesize=5.0 K 2024-11-11T02:05:00,213 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/225fba670a48464dbda22c3baab5b05c as hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/225fba670a48464dbda22c3baab5b05c 2024-11-11T02:05:00,218 INFO [M:0;370bc2ade342:43975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44375/user/jenkins/test-data/2baee0d3-4d8f-c032-b8ec-5c81646f9911/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/225fba670a48464dbda22c3baab5b05c, entries=1, sequenceid=29, filesize=4.9 K 2024-11-11T02:05:00,219 INFO [M:0;370bc2ade342:43975 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 120ms, sequenceid=29, compaction requested=false 2024-11-11T02:05:00,220 INFO [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T02:05:00,220 DEBUG [M:0;370bc2ade342:43975 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731290700099Disabling compacts and flushes for region at 1731290700099Disabling writes for close at 1731290700099Obtaining lock to block concurrent updates at 1731290700099Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731290700099Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731290700100 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731290700100Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731290700100Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731290700117 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731290700117Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731290700126 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731290700140 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731290700140Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731290700149 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731290700163 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731290700163Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731290700173 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731290700186 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731290700186Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75826c1d: reopening flushed file at 1731290700196 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ae5c8d8: reopening flushed file at 1731290700201 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46b7428e: reopening flushed file at 1731290700207 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e425987: reopening flushed file at 1731290700212 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 120ms, sequenceid=29, compaction requested=false at 1731290700219 (+7 ms)Writing region close event to WAL at 1731290700220 (+1 ms)Closed at 1731290700220 2024-11-11T02:05:00,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38793 is added to blk_1073741830_1006 (size=10311) 2024-11-11T02:05:00,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37641 is added to blk_1073741830_1006 (size=10311) 2024-11-11T02:05:00,223 INFO [M:0;370bc2ade342:43975 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T02:05:00,223 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T02:05:00,223 INFO [M:0;370bc2ade342:43975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43975 2024-11-11T02:05:00,223 INFO [M:0;370bc2ade342:43975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T02:05:00,325 INFO [M:0;370bc2ade342:43975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T02:05:00,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:05:00,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43975-0x1002c86ef3b0000, quorum=127.0.0.1:55650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T02:05:00,330 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@342910ae{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:05:00,330 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e31f159{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:05:00,331 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:05:00,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38496a94{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:05:00,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b42b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,STOPPED} 2024-11-11T02:05:00,334 WARN [BP-1010209951-172.17.0.2-1731290697893 heartbeating to localhost/127.0.0.1:44375 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:05:00,334 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:05:00,334 WARN [BP-1010209951-172.17.0.2-1731290697893 heartbeating to localhost/127.0.0.1:44375 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1010209951-172.17.0.2-1731290697893 (Datanode Uuid 6efd4ca3-7cdc-42e5-ae89-3e80740f97c3) service to localhost/127.0.0.1:44375 2024-11-11T02:05:00,334 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:05:00,335 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data3/current/BP-1010209951-172.17.0.2-1731290697893 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:05:00,335 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data4/current/BP-1010209951-172.17.0.2-1731290697893 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:05:00,335 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:05:00,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c698367{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T02:05:00,337 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8c3d086{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:05:00,337 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:05:00,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@353826c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:05:00,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e4a3730{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,STOPPED} 2024-11-11T02:05:00,338 WARN [BP-1010209951-172.17.0.2-1731290697893 heartbeating to localhost/127.0.0.1:44375 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T02:05:00,338 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T02:05:00,338 WARN [BP-1010209951-172.17.0.2-1731290697893 heartbeating to localhost/127.0.0.1:44375 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1010209951-172.17.0.2-1731290697893 (Datanode Uuid b0aee3d7-2368-4964-9650-306ff12894c1) service to localhost/127.0.0.1:44375 2024-11-11T02:05:00,338 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T02:05:00,339 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data1/current/BP-1010209951-172.17.0.2-1731290697893 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:05:00,339 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/cluster_820a7088-838c-a5ea-ae71-62d315998c97/data/data2/current/BP-1010209951-172.17.0.2-1731290697893 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T02:05:00,339 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T02:05:00,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@750e25b7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T02:05:00,344 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68c359c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T02:05:00,344 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T02:05:00,344 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4685deda{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T02:05:00,344 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f4e4670{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/889ab4aa-a0b6-84cf-d5a5-dbb5ccce3d8c/hadoop.log.dir/,STOPPED} 2024-11-11T02:05:00,350 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T02:05:00,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T02:05:00,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRollOnNothingWritten Thread=135 (was 111) - Thread LEAK? -, OpenFileDescriptor=507 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=61 (was 49) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5884 (was 5925)