2024-11-23 18:50:06,548 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-23 18:50:06,560 main DEBUG Took 0.009928 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 18:50:06,560 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 18:50:06,561 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 18:50:06,562 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 18:50:06,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,569 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 18:50:06,580 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,582 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,583 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,585 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,585 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,587 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,587 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,588 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,589 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,591 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:50:06,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,593 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 18:50:06,594 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:50:06,595 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 18:50:06,597 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 18:50:06,598 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 18:50:06,599 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 18:50:06,599 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 18:50:06,607 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 18:50:06,609 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 18:50:06,611 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 18:50:06,611 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 18:50:06,612 main DEBUG createAppenders(={Console}) 2024-11-23 18:50:06,613 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-23 18:50:06,613 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-23 18:50:06,613 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-23 18:50:06,614 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 18:50:06,614 main DEBUG OutputStream closed 2024-11-23 18:50:06,614 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 18:50:06,615 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 18:50:06,615 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-23 18:50:06,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 18:50:06,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 18:50:06,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 18:50:06,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 18:50:06,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 18:50:06,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 18:50:06,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 18:50:06,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 18:50:06,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 18:50:06,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 18:50:06,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 18:50:06,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 18:50:06,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 18:50:06,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 18:50:06,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 18:50:06,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 18:50:06,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 18:50:06,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 18:50:06,695 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 18:50:06,696 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-23 18:50:06,696 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 18:50:06,697 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-23T18:50:06,923 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c 2024-11-23 18:50:06,926 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 18:50:06,926 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T18:50:06,934 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncLogRolling timeout: 13 mins 2024-11-23T18:50:06,968 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=9669 2024-11-23T18:50:06,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:50:06,992 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae, deleteOnExit=true 2024-11-23T18:50:06,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:50:06,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/test.cache.data in system properties and HBase conf 2024-11-23T18:50:06,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:50:06,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:50:06,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:50:06,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:50:06,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:50:07,076 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T18:50:07,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:50:07,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:50:07,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:50:07,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:50:07,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:50:07,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:50:07,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:50:07,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:50:07,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:50:07,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:50:07,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:50:07,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:50:07,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:50:07,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:50:07,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:50:08,246 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T18:50:08,316 INFO [Time-limited test {}] log.Log(170): Logging initialized @2465ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T18:50:08,382 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:50:08,438 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:50:08,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:50:08,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:50:08,456 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:50:08,466 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:50:08,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20abc9e4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:50:08,470 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6932e96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:50:08,631 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@612e4b2b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/java.io.tmpdir/jetty-localhost-40841-hadoop-hdfs-3_4_1-tests_jar-_-any-2817328586883470564/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:50:08,640 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@65270512{HTTP/1.1, (http/1.1)}{localhost:40841} 2024-11-23T18:50:08,640 INFO [Time-limited test {}] server.Server(415): Started @2790ms 2024-11-23T18:50:09,175 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:50:09,182 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:50:09,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:50:09,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:50:09,183 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:50:09,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2973dc0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:50:09,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49b96e54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:50:09,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@790fbcfe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/java.io.tmpdir/jetty-localhost-39669-hadoop-hdfs-3_4_1-tests_jar-_-any-12708719905342087477/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:50:09,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e3c8436{HTTP/1.1, (http/1.1)}{localhost:39669} 2024-11-23T18:50:09,286 INFO [Time-limited test {}] server.Server(415): Started @3436ms 2024-11-23T18:50:09,338 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:50:09,448 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:50:09,456 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:50:09,461 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:50:09,461 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:50:09,461 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:50:09,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d434c3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:50:09,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@775a6b37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:50:09,593 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b9fe80c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/java.io.tmpdir/jetty-localhost-45145-hadoop-hdfs-3_4_1-tests_jar-_-any-18006836757026582710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:50:09,593 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@431c6310{HTTP/1.1, (http/1.1)}{localhost:45145} 2024-11-23T18:50:09,594 INFO [Time-limited test {}] server.Server(415): Started @3743ms 2024-11-23T18:50:09,596 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:50:11,132 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data2/current/BP-906495530-172.17.0.2-1732387807734/current, will proceed with Du for space computation calculation, 2024-11-23T18:50:11,132 WARN [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data3/current/BP-906495530-172.17.0.2-1732387807734/current, will proceed with Du for space computation calculation, 2024-11-23T18:50:11,132 WARN [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data4/current/BP-906495530-172.17.0.2-1732387807734/current, will proceed with Du for space computation calculation, 2024-11-23T18:50:11,132 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data1/current/BP-906495530-172.17.0.2-1732387807734/current, will proceed with Du for space computation calculation, 2024-11-23T18:50:11,164 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:50:11,164 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:50:11,217 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca54d2c6a8c5d73a with lease ID 0xd7ebf6cf2e1099ff: Processing first storage report for DS-7762970d-d5b7-4a1c-a058-42c6654fb528 from datanode DatanodeRegistration(127.0.0.1:45805, datanodeUuid=32cb89c9-65c8-46b2-8c44-ca2ce79db00f, infoPort=39333, infoSecurePort=0, ipcPort=38169, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734) 2024-11-23T18:50:11,218 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca54d2c6a8c5d73a with lease ID 0xd7ebf6cf2e1099ff: from storage DS-7762970d-d5b7-4a1c-a058-42c6654fb528 node DatanodeRegistration(127.0.0.1:45805, datanodeUuid=32cb89c9-65c8-46b2-8c44-ca2ce79db00f, infoPort=39333, infoSecurePort=0, ipcPort=38169, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T18:50:11,219 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x98cf302bb0e56efe with lease ID 0xd7ebf6cf2e1099fe: Processing first storage report for DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6 from datanode DatanodeRegistration(127.0.0.1:45419, datanodeUuid=944fb8b8-f741-41d6-aee9-9ee6cc03c038, infoPort=36375, infoSecurePort=0, ipcPort=37225, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734) 2024-11-23T18:50:11,219 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x98cf302bb0e56efe with lease ID 0xd7ebf6cf2e1099fe: from storage DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6 node DatanodeRegistration(127.0.0.1:45419, datanodeUuid=944fb8b8-f741-41d6-aee9-9ee6cc03c038, infoPort=36375, infoSecurePort=0, ipcPort=37225, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T18:50:11,219 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca54d2c6a8c5d73a with lease ID 0xd7ebf6cf2e1099ff: Processing first storage report for DS-1ce4906e-02c2-436e-985a-a4cdea54f060 from datanode DatanodeRegistration(127.0.0.1:45805, datanodeUuid=32cb89c9-65c8-46b2-8c44-ca2ce79db00f, infoPort=39333, infoSecurePort=0, ipcPort=38169, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734) 2024-11-23T18:50:11,220 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca54d2c6a8c5d73a with lease ID 0xd7ebf6cf2e1099ff: from storage DS-1ce4906e-02c2-436e-985a-a4cdea54f060 node DatanodeRegistration(127.0.0.1:45805, datanodeUuid=32cb89c9-65c8-46b2-8c44-ca2ce79db00f, infoPort=39333, infoSecurePort=0, ipcPort=38169, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:50:11,220 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x98cf302bb0e56efe with lease ID 0xd7ebf6cf2e1099fe: Processing first storage report for DS-7104092c-a6f4-4aad-b13c-53c422645935 from datanode DatanodeRegistration(127.0.0.1:45419, datanodeUuid=944fb8b8-f741-41d6-aee9-9ee6cc03c038, infoPort=36375, infoSecurePort=0, ipcPort=37225, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734) 2024-11-23T18:50:11,220 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x98cf302bb0e56efe with lease ID 0xd7ebf6cf2e1099fe: from storage DS-7104092c-a6f4-4aad-b13c-53c422645935 node DatanodeRegistration(127.0.0.1:45419, datanodeUuid=944fb8b8-f741-41d6-aee9-9ee6cc03c038, infoPort=36375, infoSecurePort=0, ipcPort=37225, storageInfo=lv=-57;cid=testClusterID;nsid=1154659632;c=1732387807734), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T18:50:11,242 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c 2024-11-23T18:50:11,305 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/zookeeper_0, clientPort=51904, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:50:11,315 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51904 2024-11-23T18:50:11,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:11,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:11,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:50:11,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:50:11,948 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e with version=8 2024-11-23T18:50:11,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase-staging 2024-11-23T18:50:12,038 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T18:50:12,263 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:50:12,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,277 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:50:12,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:50:12,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:50:12,467 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T18:50:12,476 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T18:50:12,479 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:50:12,504 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 64654 (auto-detected) 2024-11-23T18:50:12,505 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T18:50:12,525 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38481 2024-11-23T18:50:12,547 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38481 connecting to ZooKeeper ensemble=127.0.0.1:51904 2024-11-23T18:50:12,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384810x0, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:50:12,732 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38481-0x10169082b830000 connected 2024-11-23T18:50:12,816 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:12,818 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:12,827 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:50:12,831 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e, hbase.cluster.distributed=false 2024-11-23T18:50:12,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:50:12,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38481 2024-11-23T18:50:12,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38481 2024-11-23T18:50:12,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38481 2024-11-23T18:50:12,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38481 2024-11-23T18:50:12,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38481 2024-11-23T18:50:12,954 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:50:12,955 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,956 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,956 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:50:12,956 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:50:12,956 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:50:12,958 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:50:12,961 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:50:12,962 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43897 2024-11-23T18:50:12,964 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43897 connecting to ZooKeeper ensemble=127.0.0.1:51904 2024-11-23T18:50:12,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:12,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:13,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438970x0, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:50:13,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438970x0, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:50:13,005 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43897-0x10169082b830001 connected 2024-11-23T18:50:13,008 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:50:13,015 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:50:13,018 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:50:13,023 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:50:13,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43897 2024-11-23T18:50:13,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43897 2024-11-23T18:50:13,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43897 2024-11-23T18:50:13,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43897 2024-11-23T18:50:13,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43897 2024-11-23T18:50:13,042 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:38481 2024-11-23T18:50:13,043 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:50:13,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:50:13,058 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:50:13,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,089 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:50:13,090 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,38481,1732387812111 from backup master directory 2024-11-23T18:50:13,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:50:13,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:50:13,099 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:50:13,099 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,101 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T18:50:13,103 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T18:50:13,155 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase.id] with ID: ce6f841a-2303-4b59-aa0f-7d4132897065 2024-11-23T18:50:13,155 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/.tmp/hbase.id 2024-11-23T18:50:13,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:50:13,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:50:13,169 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/.tmp/hbase.id]:[hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase.id] 2024-11-23T18:50:13,213 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:13,217 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:50:13,235 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-23T18:50:13,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:50:13,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:50:13,277 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:50:13,279 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:50:13,285 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:50:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:50:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:50:13,334 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store 2024-11-23T18:50:13,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:50:13,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:50:13,361 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T18:50:13,365 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:13,367 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:50:13,367 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:13,367 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:13,369 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:50:13,369 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:13,369 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:50:13,371 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387813366Disabling compacts and flushes for region at 1732387813366Disabling writes for close at 1732387813369 (+3 ms)Writing region close event to WAL at 1732387813369Closed at 1732387813369 2024-11-23T18:50:13,373 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/.initializing 2024-11-23T18:50:13,374 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/WALs/2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,381 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:50:13,396 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C38481%2C1732387812111, suffix=, logDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/WALs/2eda60943ea1,38481,1732387812111, archiveDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/oldWALs, maxLogs=10 2024-11-23T18:50:13,423 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/WALs/2eda60943ea1,38481,1732387812111/2eda60943ea1%2C38481%2C1732387812111.1732387813400, exclude list is [], retry=0 2024-11-23T18:50:13,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:50:13,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:13,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:13,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T18:50:13,487 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/WALs/2eda60943ea1,38481,1732387812111/2eda60943ea1%2C38481%2C1732387812111.1732387813400 2024-11-23T18:50:13,488 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36375:36375),(127.0.0.1/127.0.0.1:39333:39333)] 2024-11-23T18:50:13,489 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:50:13,489 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:13,492 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,493 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:50:13,558 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:13,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:13,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:50:13,565 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:13,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:13,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:50:13,570 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:13,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:13,571 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:50:13,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:13,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:13,577 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,581 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,582 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,588 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,589 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,593 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:50:13,597 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:50:13,603 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:50:13,604 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834226, jitterRate=0.0607735812664032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:50:13,612 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387813505Initializing all the Stores at 1732387813508 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387813508Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387813509 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387813509Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387813510 (+1 ms)Cleaning up temporary data from old regions at 1732387813589 (+79 ms)Region opened successfully at 1732387813612 (+23 ms) 2024-11-23T18:50:13,614 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:50:13,648 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f71c5e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:50:13,675 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:50:13,685 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:50:13,685 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:50:13,688 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:50:13,689 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T18:50:13,693 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T18:50:13,693 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:50:13,716 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:50:13,724 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:50:13,772 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:50:13,774 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:50:13,776 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:50:13,782 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:50:13,784 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:50:13,788 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:50:13,792 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:50:13,794 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:50:13,803 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:50:13,819 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:50:13,824 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:50:13,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:50:13,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:50:13,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,838 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,38481,1732387812111, sessionid=0x10169082b830000, setting cluster-up flag (Was=false) 2024-11-23T18:50:13,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,898 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:50:13,900 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:13,951 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:50:13,953 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,38481,1732387812111 2024-11-23T18:50:13,958 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:50:14,031 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(746): ClusterId : ce6f841a-2303-4b59-aa0f-7d4132897065 2024-11-23T18:50:14,031 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:50:14,033 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:50:14,040 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:50:14,047 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:50:14,048 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:50:14,048 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:50:14,053 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,38481,1732387812111 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:50:14,057 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:50:14,058 DEBUG [RS:0;2eda60943ea1:43897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79301180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:50:14,060 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,061 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:50:14,061 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,062 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387844062 2024-11-23T18:50:14,063 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:50:14,064 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:50:14,066 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:50:14,067 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:50:14,068 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:50:14,068 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:50:14,069 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:50:14,069 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:50:14,070 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,074 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,074 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:50:14,075 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:43897 2024-11-23T18:50:14,076 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:50:14,077 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:50:14,078 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:50:14,078 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:50:14,078 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:50:14,079 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:50:14,080 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:50:14,080 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:50:14,082 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,38481,1732387812111 with port=43897, startcode=1732387812922 2024-11-23T18:50:14,090 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387814081,5,FailOnTimeoutGroup] 2024-11-23T18:50:14,092 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387814091,5,FailOnTimeoutGroup] 2024-11-23T18:50:14,092 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,092 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:50:14,093 DEBUG [RS:0;2eda60943ea1:43897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:50:14,094 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,094 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:50:14,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:50:14,101 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:50:14,101 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e 2024-11-23T18:50:14,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:50:14,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:50:14,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:14,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:50:14,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52323, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:50:14,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:50:14,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:50:14,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:50:14,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,136 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38481 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:50:14,139 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38481 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:50:14,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:50:14,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:50:14,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:50:14,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740 2024-11-23T18:50:14,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740 2024-11-23T18:50:14,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:50:14,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:50:14,154 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e 2024-11-23T18:50:14,154 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34337 2024-11-23T18:50:14,154 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:50:14,154 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:50:14,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:50:14,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:50:14,162 DEBUG [RS:0;2eda60943ea1:43897 {}] zookeeper.ZKUtil(111): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,162 WARN [RS:0;2eda60943ea1:43897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:50:14,162 INFO [RS:0;2eda60943ea1:43897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:50:14,162 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:50:14,166 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846583, jitterRate=0.07648603618144989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:50:14,166 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,43897,1732387812922] 2024-11-23T18:50:14,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387814125Initializing all the Stores at 1732387814126 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814127 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814127Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387814127Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814127Cleaning up temporary data from old regions at 1732387814153 (+26 ms)Region opened successfully at 1732387814170 (+17 ms) 2024-11-23T18:50:14,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:50:14,171 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:50:14,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:50:14,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:50:14,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:50:14,173 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:50:14,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387814171Disabling compacts and flushes for region at 1732387814171Disabling writes for close at 1732387814171Writing region close event to WAL at 1732387814172 (+1 ms)Closed at 1732387814173 (+1 ms) 2024-11-23T18:50:14,179 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:50:14,179 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:50:14,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:50:14,192 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:50:14,197 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:50:14,200 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:50:14,209 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:50:14,213 INFO [RS:0;2eda60943ea1:43897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:50:14,214 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,215 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:50:14,220 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:50:14,221 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,222 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:50:14,223 DEBUG [RS:0;2eda60943ea1:43897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:50:14,224 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,224 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,225 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,225 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,225 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,225 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43897,1732387812922-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:50:14,243 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:50:14,244 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43897,1732387812922-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,245 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,245 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.Replication(171): 2eda60943ea1,43897,1732387812922 started 2024-11-23T18:50:14,261 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:14,262 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,43897,1732387812922, RpcServer on 2eda60943ea1/172.17.0.2:43897, sessionid=0x10169082b830001 2024-11-23T18:50:14,262 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:50:14,262 DEBUG [RS:0;2eda60943ea1:43897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,263 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43897,1732387812922' 2024-11-23T18:50:14,263 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:50:14,264 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:50:14,264 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:50:14,264 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:50:14,265 DEBUG [RS:0;2eda60943ea1:43897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,265 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43897,1732387812922' 2024-11-23T18:50:14,265 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:50:14,265 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:50:14,266 DEBUG [RS:0;2eda60943ea1:43897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:50:14,266 INFO [RS:0;2eda60943ea1:43897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:50:14,266 INFO [RS:0;2eda60943ea1:43897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:50:14,351 WARN [2eda60943ea1:38481 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:50:14,370 INFO [RS:0;2eda60943ea1:43897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:50:14,373 INFO [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43897%2C1732387812922, suffix=, logDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922, archiveDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs, maxLogs=32 2024-11-23T18:50:14,391 DEBUG [RS:0;2eda60943ea1:43897 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387814376, exclude list is [], retry=0 2024-11-23T18:50:14,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:14,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:14,401 INFO [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387814376 2024-11-23T18:50:14,402 DEBUG [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36375:36375),(127.0.0.1/127.0.0.1:39333:39333)] 2024-11-23T18:50:14,604 DEBUG [2eda60943ea1:38481 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T18:50:14,614 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,621 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43897,1732387812922, state=OPENING 2024-11-23T18:50:14,666 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:50:14,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:14,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:50:14,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:50:14,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:50:14,680 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:50:14,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43897,1732387812922}] 2024-11-23T18:50:14,865 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:50:14,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40091, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:50:14,876 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:50:14,877 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:50:14,877 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:50:14,880 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43897%2C1732387812922.meta, suffix=.meta, logDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922, archiveDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs, maxLogs=32 2024-11-23T18:50:14,895 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.meta.1732387814882.meta, exclude list is [], retry=0 2024-11-23T18:50:14,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:14,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:14,902 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.meta.1732387814882.meta 2024-11-23T18:50:14,902 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36375:36375),(127.0.0.1/127.0.0.1:39333:39333)] 2024-11-23T18:50:14,903 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:50:14,905 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:50:14,907 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:50:14,912 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:50:14,916 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:50:14,916 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:14,917 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:50:14,917 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:50:14,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:50:14,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:50:14,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:50:14,924 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:50:14,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:50:14,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:50:14,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:50:14,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:50:14,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:14,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:50:14,930 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:50:14,931 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740 2024-11-23T18:50:14,934 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740 2024-11-23T18:50:14,937 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:50:14,937 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:50:14,938 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:50:14,941 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:50:14,942 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=853104, jitterRate=0.08477786183357239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:50:14,943 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:50:14,944 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387814917Writing region info on filesystem at 1732387814917Initializing all the Stores at 1732387814919 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814919Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814919Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387814919Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387814919Cleaning up temporary data from old regions at 1732387814937 (+18 ms)Running coprocessor post-open hooks at 1732387814943 (+6 ms)Region opened successfully at 1732387814944 (+1 ms) 2024-11-23T18:50:14,951 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387814855 2024-11-23T18:50:14,962 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:50:14,962 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:50:14,964 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:14,966 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43897,1732387812922, state=OPEN 2024-11-23T18:50:15,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:50:15,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:50:15,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:50:15,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:50:15,061 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:15,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:50:15,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43897,1732387812922 in 378 msec 2024-11-23T18:50:15,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:50:15,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 881 msec 2024-11-23T18:50:15,075 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:50:15,076 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:50:15,093 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:50:15,094 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43897,1732387812922, seqNum=-1] 2024-11-23T18:50:15,112 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:50:15,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51625, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:50:15,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1460 sec 2024-11-23T18:50:15,133 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387815133, completionTime=-1 2024-11-23T18:50:15,136 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T18:50:15,136 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:50:15,162 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T18:50:15,162 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732387875162 2024-11-23T18:50:15,162 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732387935162 2024-11-23T18:50:15,162 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-23T18:50:15,164 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,164 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,164 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,166 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:38481, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,166 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,166 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,174 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:50:15,195 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.095sec 2024-11-23T18:50:15,196 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:50:15,197 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:50:15,198 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:50:15,198 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:50:15,198 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:50:15,199 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:50:15,200 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:50:15,208 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:50:15,210 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:50:15,210 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,38481,1732387812111-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:50:15,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5379a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:50:15,244 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T18:50:15,244 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T18:50:15,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,38481,-1 for getting cluster id 2024-11-23T18:50:15,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:50:15,260 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce6f841a-2303-4b59-aa0f-7d4132897065' 2024-11-23T18:50:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:50:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce6f841a-2303-4b59-aa0f-7d4132897065" 2024-11-23T18:50:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3511359d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:50:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,38481,-1] 2024-11-23T18:50:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:50:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:50:15,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:50:15,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b54472, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:50:15,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:50:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43897,1732387812922, seqNum=-1] 2024-11-23T18:50:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:50:15,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:50:15,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,38481,1732387812111 2024-11-23T18:50:15,310 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:50:15,318 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-23T18:50:15,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:50:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,38481,1732387812111 2024-11-23T18:50:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@fb2a85e 2024-11-23T18:50:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:50:15,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:50:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-23T18:50:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-23T18:50:15,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:50:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-23T18:50:15,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:50:15,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-23T18:50:15,355 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:15,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:50:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:50:15,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741835_1011 (size=389) 2024-11-23T18:50:15,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741835_1011 (size=389) 2024-11-23T18:50:15,374 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cdd6d97d9dd019f1f530d449189dbae1, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e 2024-11-23T18:50:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741836_1012 (size=72) 2024-11-23T18:50:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741836_1012 (size=72) 2024-11-23T18:50:15,388 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:15,389 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing cdd6d97d9dd019f1f530d449189dbae1, disabling compactions & flushes 2024-11-23T18:50:15,389 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,389 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,389 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. after waiting 0 ms 2024-11-23T18:50:15,389 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,389 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,389 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for cdd6d97d9dd019f1f530d449189dbae1: Waiting for close lock at 1732387815389Disabling compacts and flushes for region at 1732387815389Disabling writes for close at 1732387815389Writing region close event to WAL at 1732387815389Closed at 1732387815389 2024-11-23T18:50:15,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:50:15,397 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732387815392"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387815392"}]},"ts":"1732387815392"} 2024-11-23T18:50:15,402 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:50:15,404 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:50:15,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387815404"}]},"ts":"1732387815404"} 2024-11-23T18:50:15,412 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-23T18:50:15,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=cdd6d97d9dd019f1f530d449189dbae1, ASSIGN}] 2024-11-23T18:50:15,417 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=cdd6d97d9dd019f1f530d449189dbae1, ASSIGN 2024-11-23T18:50:15,419 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=cdd6d97d9dd019f1f530d449189dbae1, ASSIGN; state=OFFLINE, location=2eda60943ea1,43897,1732387812922; forceNewPlan=false, retain=false 2024-11-23T18:50:15,570 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cdd6d97d9dd019f1f530d449189dbae1, regionState=OPENING, regionLocation=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:15,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=cdd6d97d9dd019f1f530d449189dbae1, ASSIGN because future has completed 2024-11-23T18:50:15,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd6d97d9dd019f1f530d449189dbae1, server=2eda60943ea1,43897,1732387812922}] 2024-11-23T18:50:15,736 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,736 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cdd6d97d9dd019f1f530d449189dbae1, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:50:15,737 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,737 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:50:15,737 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,737 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,740 INFO [StoreOpener-cdd6d97d9dd019f1f530d449189dbae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,743 INFO [StoreOpener-cdd6d97d9dd019f1f530d449189dbae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdd6d97d9dd019f1f530d449189dbae1 columnFamilyName info 2024-11-23T18:50:15,743 DEBUG [StoreOpener-cdd6d97d9dd019f1f530d449189dbae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:50:15,744 INFO [StoreOpener-cdd6d97d9dd019f1f530d449189dbae1-1 {}] regionserver.HStore(327): Store=cdd6d97d9dd019f1f530d449189dbae1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:50:15,744 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,746 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,747 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,748 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,748 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,751 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,755 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:50:15,756 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cdd6d97d9dd019f1f530d449189dbae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715453, jitterRate=-0.09025482833385468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:50:15,757 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:15,758 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cdd6d97d9dd019f1f530d449189dbae1: Running coprocessor pre-open hook at 1732387815737Writing region info on filesystem at 1732387815737Initializing all the Stores at 1732387815739 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387815740 (+1 ms)Cleaning up temporary data from old regions at 1732387815748 (+8 ms)Running coprocessor post-open hooks at 1732387815757 (+9 ms)Region opened successfully at 1732387815758 (+1 ms) 2024-11-23T18:50:15,760 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1., pid=6, masterSystemTime=1732387815729 2024-11-23T18:50:15,764 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,764 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:15,765 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cdd6d97d9dd019f1f530d449189dbae1, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,43897,1732387812922 2024-11-23T18:50:15,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd6d97d9dd019f1f530d449189dbae1, server=2eda60943ea1,43897,1732387812922 because future has completed 2024-11-23T18:50:15,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:50:15,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cdd6d97d9dd019f1f530d449189dbae1, server=2eda60943ea1,43897,1732387812922 in 197 msec 2024-11-23T18:50:15,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:50:15,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=cdd6d97d9dd019f1f530d449189dbae1, ASSIGN in 362 msec 2024-11-23T18:50:15,782 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:50:15,783 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387815782"}]},"ts":"1732387815782"} 2024-11-23T18:50:15,786 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-23T18:50:15,788 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:50:15,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 445 msec 2024-11-23T18:50:20,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:50:20,506 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:50:20,508 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-23T18:50:22,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:50:22,464 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T18:50:22,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-23T18:50:22,465 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-23T18:50:22,466 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:50:22,466 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T18:50:22,466 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T18:50:22,466 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T18:50:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:50:25,463 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-23T18:50:25,466 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-23T18:50:25,473 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-23T18:50:25,474 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:50:25,489 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387825475, exclude list is [], retry=0 2024-11-23T18:50:25,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:25,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:25,496 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387814376 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387825475 2024-11-23T18:50:25,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36375:36375),(127.0.0.1/127.0.0.1:39333:39333)] 2024-11-23T18:50:25,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387814376 is not closed yet, will try archiving it next time 2024-11-23T18:50:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741833_1009 (size=451) 2024-11-23T18:50:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741833_1009 (size=451) 2024-11-23T18:50:25,504 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387814376 to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs/2eda60943ea1%2C43897%2C1732387812922.1732387814376 2024-11-23T18:50:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1., hostname=2eda60943ea1,43897,1732387812922, seqNum=2] 2024-11-23T18:50:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43897 {}] regionserver.HRegion(8855): Flush requested on cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:50:39,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdd6d97d9dd019f1f530d449189dbae1 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:50:39,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/667dc4c154ae4938809992eca8003258 is 1055, key is row0001/info:/1732387825508/Put/seqid=0 2024-11-23T18:50:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741838_1014 (size=13386) 2024-11-23T18:50:39,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741838_1014 (size=13386) 2024-11-23T18:50:39,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/667dc4c154ae4938809992eca8003258 2024-11-23T18:50:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/667dc4c154ae4938809992eca8003258 as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/667dc4c154ae4938809992eca8003258 2024-11-23T18:50:39,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/667dc4c154ae4938809992eca8003258, entries=8, sequenceid=12, filesize=13.1 K 2024-11-23T18:50:39,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=0 B/0 for cdd6d97d9dd019f1f530d449189dbae1 in 128ms, sequenceid=12, compaction requested=false 2024-11-23T18:50:39,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdd6d97d9dd019f1f530d449189dbae1: 2024-11-23T18:50:41,240 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:50:45,575 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387845557, exclude list is [], retry=0 2024-11-23T18:50:45,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:45,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:45,582 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387825475 with entries=12, filesize=11.86 KB; new WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387845557 2024-11-23T18:50:45,582 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39333:39333),(127.0.0.1/127.0.0.1:36375:36375)] 2024-11-23T18:50:45,582 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387825475 is not closed yet, will try archiving it next time 2024-11-23T18:50:45,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741837_1013 (size=12149) 2024-11-23T18:50:45,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741837_1013 (size=12149) 2024-11-23T18:50:50,588 INFO [AsyncFSWAL-0-hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e-prefix:2eda60943ea1,43897,1732387812922 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5003 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK], DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK]] 2024-11-23T18:50:50,590 WARN [AsyncFSWAL-0-hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e-prefix:2eda60943ea1,43897,1732387812922 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5003 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK], DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK]] 2024-11-23T18:50:50,591 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2eda60943ea1%2C43897%2C1732387812922:(num 1732387845557) roll requested 2024-11-23T18:50:50,605 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387850591, exclude list is [], retry=0 2024-11-23T18:50:50,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:50,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:50,611 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387845557 with entries=1, filesize=1.20 KB; new WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387850591 2024-11-23T18:50:50,611 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39333:39333),(127.0.0.1/127.0.0.1:36375:36375)] 2024-11-23T18:50:50,611 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387845557 is not closed yet, will try archiving it next time 2024-11-23T18:50:50,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741839_1015 (size=1233) 2024-11-23T18:50:50,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741839_1015 (size=1233) 2024-11-23T18:50:50,614 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387845557 not finished, retry = 0 2024-11-23T18:50:52,590 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:50:52,605 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387852590, exclude list is [], retry=0 2024-11-23T18:50:52,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45805,DS-7762970d-d5b7-4a1c-a058-42c6654fb528,DISK] 2024-11-23T18:50:52,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45419,DS-d9eb4d02-55ab-4e4f-aabb-175ee1e1c8d6,DISK] 2024-11-23T18:50:52,611 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387850591 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387852590 2024-11-23T18:50:52,612 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36375:36375),(127.0.0.1/127.0.0.1:39333:39333)] 2024-11-23T18:50:52,612 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387850591 is not closed yet, will try archiving it next time 2024-11-23T18:50:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741840_1016 (size=93) 2024-11-23T18:50:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741840_1016 (size=93) 2024-11-23T18:50:52,615 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387850591 to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs/2eda60943ea1%2C43897%2C1732387812922.1732387850591 2024-11-23T18:50:55,623 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T18:50:55,623 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T18:51:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43897 {}] regionserver.HRegion(8855): Flush requested on cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:51:00,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdd6d97d9dd019f1f530d449189dbae1 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:51:00,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/cb19810ead15415980792c4b158c05d8 is 1055, key is row0009/info:/1732387841547/Put/seqid=0 2024-11-23T18:51:00,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741842_1018 (size=13386) 2024-11-23T18:51:00,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741842_1018 (size=13386) 2024-11-23T18:51:00,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/cb19810ead15415980792c4b158c05d8 2024-11-23T18:51:00,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/cb19810ead15415980792c4b158c05d8 as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/cb19810ead15415980792c4b158c05d8 2024-11-23T18:51:00,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/cb19810ead15415980792c4b158c05d8, entries=8, sequenceid=23, filesize=13.1 K 2024-11-23T18:51:00,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=0 B/0 for cdd6d97d9dd019f1f530d449189dbae1 in 37ms, sequenceid=23, compaction requested=false 2024-11-23T18:51:00,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdd6d97d9dd019f1f530d449189dbae1: 2024-11-23T18:51:00,667 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.1 K, sizeToCheck=16.0 K 2024-11-23T18:51:00,667 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:51:00,667 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/667dc4c154ae4938809992eca8003258 because midkey is the same as first or last row 2024-11-23T18:51:00,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region cdd6d97d9dd019f1f530d449189dbae1, had cached 0 bytes from a total of 26772 2024-11-23T18:51:11,240 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:51:12,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:51:12,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:51:12,652 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:12,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:12,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:12,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:51:12,663 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:51:12,663 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1012030733, stopped=false 2024-11-23T18:51:12,663 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,38481,1732387812111 2024-11-23T18:51:12,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:12,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:12,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:12,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:12,726 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:51:12,726 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:51:12,727 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:12,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:12,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:12,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:12,728 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,43897,1732387812922' ***** 2024-11-23T18:51:12,728 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:51:12,729 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:51:12,730 INFO [RS:0;2eda60943ea1:43897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:51:12,730 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:51:12,730 INFO [RS:0;2eda60943ea1:43897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:51:12,731 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(3091): Received CLOSE for cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:51:12,731 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,43897,1732387812922 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:43897. 2024-11-23T18:51:12,732 DEBUG [RS:0;2eda60943ea1:43897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:12,732 DEBUG [RS:0;2eda60943ea1:43897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:12,732 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cdd6d97d9dd019f1f530d449189dbae1, disabling compactions & flushes 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:51:12,732 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:51:12,732 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:51:12,732 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. after waiting 0 ms 2024-11-23T18:51:12,732 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:51:12,732 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:51:12,733 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cdd6d97d9dd019f1f530d449189dbae1 1/1 column families, dataSize=5.13 KB heapSize=5.76 KB 2024-11-23T18:51:12,733 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:51:12,733 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:51:12,733 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1325): Online Regions={cdd6d97d9dd019f1f530d449189dbae1=TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1., 1588230740=hbase:meta,,1.1588230740} 2024-11-23T18:51:12,733 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:51:12,733 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:51:12,733 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:51:12,733 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:51:12,733 DEBUG [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cdd6d97d9dd019f1f530d449189dbae1 2024-11-23T18:51:12,733 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-23T18:51:12,740 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/7815ac9a544447e29d8fd93e3366707d is 1055, key is row0017/info:/1732387862631/Put/seqid=0 2024-11-23T18:51:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741843_1019 (size=10222) 2024-11-23T18:51:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741843_1019 (size=10222) 2024-11-23T18:51:12,748 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/7815ac9a544447e29d8fd93e3366707d 2024-11-23T18:51:12,755 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/info/82f267a5fcd849048e2de7522e60e680 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1./info:regioninfo/1732387815765/Put/seqid=0 2024-11-23T18:51:12,760 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/.tmp/info/7815ac9a544447e29d8fd93e3366707d as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/7815ac9a544447e29d8fd93e3366707d 2024-11-23T18:51:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741844_1020 (size=7016) 2024-11-23T18:51:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741844_1020 (size=7016) 2024-11-23T18:51:12,763 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/info/82f267a5fcd849048e2de7522e60e680 2024-11-23T18:51:12,770 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/info/7815ac9a544447e29d8fd93e3366707d, entries=5, sequenceid=31, filesize=10.0 K 2024-11-23T18:51:12,771 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.13 KB/5255, heapSize ~5.74 KB/5880, currentSize=0 B/0 for cdd6d97d9dd019f1f530d449189dbae1 in 39ms, sequenceid=31, compaction requested=true 2024-11-23T18:51:12,777 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/default/TestLogRolling-testSlowSyncLogRolling/cdd6d97d9dd019f1f530d449189dbae1/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=1 2024-11-23T18:51:12,779 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:51:12,780 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cdd6d97d9dd019f1f530d449189dbae1: Waiting for close lock at 1732387872732Running coprocessor pre-close hooks at 1732387872732Disabling compacts and flushes for region at 1732387872732Disabling writes for close at 1732387872732Obtaining lock to block concurrent updates at 1732387872733 (+1 ms)Preparing flush snapshotting stores in cdd6d97d9dd019f1f530d449189dbae1 at 1732387872733Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1., syncing WAL and waiting on mvcc, flushsize=dataSize=5255, getHeapSize=5880, getOffHeapSize=0, getCellsCount=5 at 1732387872733Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. at 1732387872734 (+1 ms)Flushing cdd6d97d9dd019f1f530d449189dbae1/info: creating writer at 1732387872734Flushing cdd6d97d9dd019f1f530d449189dbae1/info: appending metadata at 1732387872740 (+6 ms)Flushing cdd6d97d9dd019f1f530d449189dbae1/info: closing flushed file at 1732387872740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3218486: reopening flushed file at 1732387872758 (+18 ms)Finished flush of dataSize ~5.13 KB/5255, heapSize ~5.74 KB/5880, currentSize=0 B/0 for cdd6d97d9dd019f1f530d449189dbae1 in 39ms, sequenceid=31, compaction requested=true at 1732387872771 (+13 ms)Writing region close event to WAL at 1732387872772 (+1 ms)Running coprocessor post-close hooks at 1732387872778 (+6 ms)Closed at 1732387872779 (+1 ms) 2024-11-23T18:51:12,780 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732387815336.cdd6d97d9dd019f1f530d449189dbae1. 2024-11-23T18:51:12,792 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/ns/63880a5bf595470d859a2be81e630aae is 43, key is default/ns:d/1732387815118/Put/seqid=0 2024-11-23T18:51:12,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741845_1021 (size=5153) 2024-11-23T18:51:12,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741845_1021 (size=5153) 2024-11-23T18:51:12,800 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/ns/63880a5bf595470d859a2be81e630aae 2024-11-23T18:51:12,823 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/table/4ad4d597835047dc8235327e15ada3da is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732387815782/Put/seqid=0 2024-11-23T18:51:12,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741846_1022 (size=5396) 2024-11-23T18:51:12,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741846_1022 (size=5396) 2024-11-23T18:51:12,830 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/table/4ad4d597835047dc8235327e15ada3da 2024-11-23T18:51:12,840 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/info/82f267a5fcd849048e2de7522e60e680 as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/info/82f267a5fcd849048e2de7522e60e680 2024-11-23T18:51:12,850 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/info/82f267a5fcd849048e2de7522e60e680, entries=10, sequenceid=11, filesize=6.9 K 2024-11-23T18:51:12,852 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/ns/63880a5bf595470d859a2be81e630aae as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/ns/63880a5bf595470d859a2be81e630aae 2024-11-23T18:51:12,861 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/ns/63880a5bf595470d859a2be81e630aae, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T18:51:12,863 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/.tmp/table/4ad4d597835047dc8235327e15ada3da as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/table/4ad4d597835047dc8235327e15ada3da 2024-11-23T18:51:12,871 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/table/4ad4d597835047dc8235327e15ada3da, entries=2, sequenceid=11, filesize=5.3 K 2024-11-23T18:51:12,873 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false 2024-11-23T18:51:12,879 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T18:51:12,880 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:51:12,880 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:12,880 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387872733Running coprocessor pre-close hooks at 1732387872733Disabling compacts and flushes for region at 1732387872733Disabling writes for close at 1732387872733Obtaining lock to block concurrent updates at 1732387872733Preparing flush snapshotting stores in 1588230740 at 1732387872733Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732387872734 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732387872735 (+1 ms)Flushing 1588230740/info: creating writer at 1732387872735Flushing 1588230740/info: appending metadata at 1732387872755 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732387872755Flushing 1588230740/ns: creating writer at 1732387872771 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732387872791 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732387872791Flushing 1588230740/table: creating writer at 1732387872808 (+17 ms)Flushing 1588230740/table: appending metadata at 1732387872823 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732387872823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@385c2dcd: reopening flushed file at 1732387872838 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e134cbf: reopening flushed file at 1732387872850 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58da678c: reopening flushed file at 1732387872862 (+12 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false at 1732387872873 (+11 ms)Writing region close event to WAL at 1732387872874 (+1 ms)Running coprocessor post-close hooks at 1732387872880 (+6 ms)Closed at 1732387872880 2024-11-23T18:51:12,880 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:12,934 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,43897,1732387812922; all regions closed. 2024-11-23T18:51:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741834_1010 (size=3066) 2024-11-23T18:51:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741834_1010 (size=3066) 2024-11-23T18:51:12,946 DEBUG [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs 2024-11-23T18:51:12,946 INFO [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43897%2C1732387812922.meta:.meta(num 1732387814882) 2024-11-23T18:51:12,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741841_1017 (size=13164) 2024-11-23T18:51:12,949 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/WALs/2eda60943ea1,43897,1732387812922/2eda60943ea1%2C43897%2C1732387812922.1732387852590 not finished, retry = 0 2024-11-23T18:51:12,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741841_1017 (size=13164) 2024-11-23T18:51:13,060 DEBUG [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/oldWALs 2024-11-23T18:51:13,060 INFO [RS:0;2eda60943ea1:43897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43897%2C1732387812922:(num 1732387852590) 2024-11-23T18:51:13,060 DEBUG [RS:0;2eda60943ea1:43897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:13,060 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:51:13,060 INFO [RS:0;2eda60943ea1:43897 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:51:13,061 INFO [RS:0;2eda60943ea1:43897 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T18:51:13,061 INFO [RS:0;2eda60943ea1:43897 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:51:13,061 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:51:13,062 INFO [RS:0;2eda60943ea1:43897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43897 2024-11-23T18:51:13,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,43897,1732387812922 2024-11-23T18:51:13,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:51:13,093 INFO [RS:0;2eda60943ea1:43897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:51:13,104 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,43897,1732387812922] 2024-11-23T18:51:13,188 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,43897,1732387812922 already deleted, retry=false 2024-11-23T18:51:13,189 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,43897,1732387812922 expired; onlineServers=0 2024-11-23T18:51:13,189 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,38481,1732387812111' ***** 2024-11-23T18:51:13,190 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:51:13,190 INFO [M:0;2eda60943ea1:38481 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:51:13,190 INFO [M:0;2eda60943ea1:38481 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:51:13,191 DEBUG [M:0;2eda60943ea1:38481 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:51:13,191 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:51:13,191 DEBUG [M:0;2eda60943ea1:38481 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:51:13,192 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387814091 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387814091,5,FailOnTimeoutGroup] 2024-11-23T18:51:13,192 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387814081 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387814081,5,FailOnTimeoutGroup] 2024-11-23T18:51:13,192 INFO [M:0;2eda60943ea1:38481 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:51:13,193 INFO [M:0;2eda60943ea1:38481 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:51:13,193 DEBUG [M:0;2eda60943ea1:38481 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:51:13,193 INFO [M:0;2eda60943ea1:38481 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:51:13,194 INFO [M:0;2eda60943ea1:38481 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:51:13,195 INFO [M:0;2eda60943ea1:38481 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:51:13,195 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:51:13,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:51:13,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:13,199 DEBUG [M:0;2eda60943ea1:38481 {}] zookeeper.ZKUtil(347): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:51:13,199 WARN [M:0;2eda60943ea1:38481 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:51:13,200 INFO [M:0;2eda60943ea1:38481 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/.lastflushedseqids 2024-11-23T18:51:13,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:13,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43897-0x10169082b830001, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:13,205 INFO [RS:0;2eda60943ea1:43897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:51:13,205 INFO [RS:0;2eda60943ea1:43897 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,43897,1732387812922; zookeeper connection closed. 2024-11-23T18:51:13,205 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@643f4f1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@643f4f1b 2024-11-23T18:51:13,206 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T18:51:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741847_1023 (size=130) 2024-11-23T18:51:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741847_1023 (size=130) 2024-11-23T18:51:13,215 INFO [M:0;2eda60943ea1:38481 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:51:13,215 INFO [M:0;2eda60943ea1:38481 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:51:13,215 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:51:13,215 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:13,215 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:13,215 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:51:13,215 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:13,215 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-11-23T18:51:13,233 DEBUG [M:0;2eda60943ea1:38481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b664552f845dbb1d4029e2aed4127 is 82, key is hbase:meta,,1/info:regioninfo/1732387814963/Put/seqid=0 2024-11-23T18:51:13,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741848_1024 (size=5672) 2024-11-23T18:51:13,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741848_1024 (size=5672) 2024-11-23T18:51:13,240 INFO [M:0;2eda60943ea1:38481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b664552f845dbb1d4029e2aed4127 2024-11-23T18:51:13,265 DEBUG [M:0;2eda60943ea1:38481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e33a23ffc63148269102cac3e56149b5 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387815791/Put/seqid=0 2024-11-23T18:51:13,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741849_1025 (size=6247) 2024-11-23T18:51:13,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741849_1025 (size=6247) 2024-11-23T18:51:13,272 INFO [M:0;2eda60943ea1:38481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e33a23ffc63148269102cac3e56149b5 2024-11-23T18:51:13,279 INFO [M:0;2eda60943ea1:38481 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e33a23ffc63148269102cac3e56149b5 2024-11-23T18:51:13,295 DEBUG [M:0;2eda60943ea1:38481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a47fdfea818425783f72225cdd7d3ca is 69, key is 2eda60943ea1,43897,1732387812922/rs:state/1732387814141/Put/seqid=0 2024-11-23T18:51:13,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741850_1026 (size=5156) 2024-11-23T18:51:13,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741850_1026 (size=5156) 2024-11-23T18:51:13,302 INFO [M:0;2eda60943ea1:38481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a47fdfea818425783f72225cdd7d3ca 2024-11-23T18:51:13,327 DEBUG [M:0;2eda60943ea1:38481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a4ece04f071a42a4ad648a468a8ac40c is 52, key is load_balancer_on/state:d/1732387815315/Put/seqid=0 2024-11-23T18:51:13,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741851_1027 (size=5056) 2024-11-23T18:51:13,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741851_1027 (size=5056) 2024-11-23T18:51:13,334 INFO [M:0;2eda60943ea1:38481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a4ece04f071a42a4ad648a468a8ac40c 2024-11-23T18:51:13,342 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b664552f845dbb1d4029e2aed4127 as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d5b664552f845dbb1d4029e2aed4127 2024-11-23T18:51:13,350 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d5b664552f845dbb1d4029e2aed4127, entries=8, sequenceid=59, filesize=5.5 K 2024-11-23T18:51:13,351 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e33a23ffc63148269102cac3e56149b5 as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e33a23ffc63148269102cac3e56149b5 2024-11-23T18:51:13,359 INFO [M:0;2eda60943ea1:38481 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e33a23ffc63148269102cac3e56149b5 2024-11-23T18:51:13,359 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e33a23ffc63148269102cac3e56149b5, entries=6, sequenceid=59, filesize=6.1 K 2024-11-23T18:51:13,360 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a47fdfea818425783f72225cdd7d3ca as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a47fdfea818425783f72225cdd7d3ca 2024-11-23T18:51:13,367 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a47fdfea818425783f72225cdd7d3ca, entries=1, sequenceid=59, filesize=5.0 K 2024-11-23T18:51:13,369 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a4ece04f071a42a4ad648a468a8ac40c as hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a4ece04f071a42a4ad648a468a8ac40c 2024-11-23T18:51:13,375 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a4ece04f071a42a4ad648a468a8ac40c, entries=1, sequenceid=59, filesize=4.9 K 2024-11-23T18:51:13,377 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 161ms, sequenceid=59, compaction requested=false 2024-11-23T18:51:13,378 INFO [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:13,378 DEBUG [M:0;2eda60943ea1:38481 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387873215Disabling compacts and flushes for region at 1732387873215Disabling writes for close at 1732387873215Obtaining lock to block concurrent updates at 1732387873215Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732387873215Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1732387873216 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732387873217 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732387873217Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732387873233 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732387873233Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732387873248 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732387873264 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732387873264Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732387873279 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732387873294 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732387873294Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732387873311 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732387873327 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732387873327Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58396505: reopening flushed file at 1732387873340 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b1103f7: reopening flushed file at 1732387873350 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41612a66: reopening flushed file at 1732387873359 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c16053: reopening flushed file at 1732387873367 (+8 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 161ms, sequenceid=59, compaction requested=false at 1732387873377 (+10 ms)Writing region close event to WAL at 1732387873378 (+1 ms)Closed at 1732387873378 2024-11-23T18:51:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45805 is added to blk_1073741830_1006 (size=27973) 2024-11-23T18:51:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45419 is added to blk_1073741830_1006 (size=27973) 2024-11-23T18:51:13,382 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:51:13,382 INFO [M:0;2eda60943ea1:38481 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:51:13,382 INFO [M:0;2eda60943ea1:38481 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38481 2024-11-23T18:51:13,383 INFO [M:0;2eda60943ea1:38481 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:51:13,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:13,488 INFO [M:0;2eda60943ea1:38481 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:51:13,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38481-0x10169082b830000, quorum=127.0.0.1:51904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:13,499 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b9fe80c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:13,501 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@431c6310{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:13,501 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:13,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@775a6b37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:13,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d434c3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:13,504 WARN [BP-906495530-172.17.0.2-1732387807734 heartbeating to localhost/127.0.0.1:34337 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:13,504 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:13,504 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:13,504 WARN [BP-906495530-172.17.0.2-1732387807734 heartbeating to localhost/127.0.0.1:34337 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906495530-172.17.0.2-1732387807734 (Datanode Uuid 944fb8b8-f741-41d6-aee9-9ee6cc03c038) service to localhost/127.0.0.1:34337 2024-11-23T18:51:13,506 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data3/current/BP-906495530-172.17.0.2-1732387807734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:13,506 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data4/current/BP-906495530-172.17.0.2-1732387807734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:13,507 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:13,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@790fbcfe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:13,511 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e3c8436{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:13,511 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:13,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49b96e54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:13,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2973dc0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:13,513 WARN [BP-906495530-172.17.0.2-1732387807734 heartbeating to localhost/127.0.0.1:34337 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:13,513 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:13,513 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:13,513 WARN [BP-906495530-172.17.0.2-1732387807734 heartbeating to localhost/127.0.0.1:34337 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906495530-172.17.0.2-1732387807734 (Datanode Uuid 32cb89c9-65c8-46b2-8c44-ca2ce79db00f) service to localhost/127.0.0.1:34337 2024-11-23T18:51:13,514 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data1/current/BP-906495530-172.17.0.2-1732387807734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:13,514 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/cluster_017e3123-8bf1-d7bb-8dd2-e2c3b2c481ae/data/data2/current/BP-906495530-172.17.0.2-1732387807734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:13,514 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:13,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@612e4b2b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:51:13,524 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@65270512{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:13,524 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:13,524 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6932e96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:13,525 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20abc9e4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:13,532 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:51:13,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T18:51:13,568 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testSlowSyncLogRolling Thread=65 (was 12) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34337 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34337 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@632e56cd java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:34337 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2eda60943ea1:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34337 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2eda60943ea1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:34337 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:34337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2eda60943ea1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Slow-sync-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:34337 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master/2eda60943ea1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=393 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=9044 (was 9669) 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRollOnDatanodeDeath Thread=66, OpenFileDescriptor=393, MaxFileDescriptor=1048576, SystemLoadAverage=144, ProcessCount=11, AvailableMemoryMB=9044 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.log.dir so I do NOT create it in target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba5910c5-4bf8-4698-4023-4bd9bb898a6c/hadoop.tmp.dir so I do NOT create it in target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86, deleteOnExit=true 2024-11-23T18:51:13,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/test.cache.data in system properties and HBase conf 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:51:13,575 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:51:13,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:51:13,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:51:13,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:51:13,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:51:13,900 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:13,906 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:13,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:13,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:13,910 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:13,911 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:13,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b1b5aec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:13,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@143ef478{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:14,019 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a9f0012{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-44497-hadoop-hdfs-3_4_1-tests_jar-_-any-2626834583244265464/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:51:14,020 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@724b4e38{HTTP/1.1, (http/1.1)}{localhost:44497} 2024-11-23T18:51:14,020 INFO [Time-limited test {}] server.Server(415): Started @68170ms 2024-11-23T18:51:14,228 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:51:14,297 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:14,302 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:14,303 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:14,303 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:14,303 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:51:14,304 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6e73dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:14,304 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52967733{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:14,403 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ff9dca{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-37731-hadoop-hdfs-3_4_1-tests_jar-_-any-6195204143242230316/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:14,404 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c6b6815{HTTP/1.1, (http/1.1)}{localhost:37731} 2024-11-23T18:51:14,404 INFO [Time-limited test {}] server.Server(415): Started @68553ms 2024-11-23T18:51:14,405 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:14,441 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:14,445 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:14,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:14,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:14,446 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:14,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f4a76c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:14,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@679249d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:14,542 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f4810cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-34961-hadoop-hdfs-3_4_1-tests_jar-_-any-9691460413341483418/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:14,543 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@18152f49{HTTP/1.1, (http/1.1)}{localhost:34961} 2024-11-23T18:51:14,543 INFO [Time-limited test {}] server.Server(415): Started @68692ms 2024-11-23T18:51:14,544 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:17,533 WARN [Thread-386 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data2/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:17,533 WARN [Thread-385 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data1/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:17,552 WARN [Thread-349 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc9e876ce1b1d7a4 with lease ID 0xd33df1f093b822b6: Processing first storage report for DS-ff9df5de-0527-4946-b726-e4069b4967c4 from datanode DatanodeRegistration(127.0.0.1:45909, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=46267, infoSecurePort=0, ipcPort=38649, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc9e876ce1b1d7a4 with lease ID 0xd33df1f093b822b6: from storage DS-ff9df5de-0527-4946-b726-e4069b4967c4 node DatanodeRegistration(127.0.0.1:45909, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=46267, infoSecurePort=0, ipcPort=38649, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc9e876ce1b1d7a4 with lease ID 0xd33df1f093b822b6: Processing first storage report for DS-f6e7b460-6311-41ac-8a1a-0d7de90ffc65 from datanode DatanodeRegistration(127.0.0.1:45909, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=46267, infoSecurePort=0, ipcPort=38649, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc9e876ce1b1d7a4 with lease ID 0xd33df1f093b822b6: from storage DS-f6e7b460-6311-41ac-8a1a-0d7de90ffc65 node DatanodeRegistration(127.0.0.1:45909, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=46267, infoSecurePort=0, ipcPort=38649, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:17,575 WARN [Thread-396 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data3/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:17,575 WARN [Thread-397 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data4/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:17,594 WARN [Thread-372 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4d84b4315ebf101 with lease ID 0xd33df1f093b822b7: Processing first storage report for DS-23f485e2-c948-49f1-beb5-7857f0b7485b from datanode DatanodeRegistration(127.0.0.1:44293, datanodeUuid=60f2802f-1415-4d27-ab88-d73cab2efbbb, infoPort=35143, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4d84b4315ebf101 with lease ID 0xd33df1f093b822b7: from storage DS-23f485e2-c948-49f1-beb5-7857f0b7485b node DatanodeRegistration(127.0.0.1:44293, datanodeUuid=60f2802f-1415-4d27-ab88-d73cab2efbbb, infoPort=35143, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4d84b4315ebf101 with lease ID 0xd33df1f093b822b7: Processing first storage report for DS-99bc3f09-e5f2-4f06-a59e-dc1587b4decd from datanode DatanodeRegistration(127.0.0.1:44293, datanodeUuid=60f2802f-1415-4d27-ab88-d73cab2efbbb, infoPort=35143, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4d84b4315ebf101 with lease ID 0xd33df1f093b822b7: from storage DS-99bc3f09-e5f2-4f06-a59e-dc1587b4decd node DatanodeRegistration(127.0.0.1:44293, datanodeUuid=60f2802f-1415-4d27-ab88-d73cab2efbbb, infoPort=35143, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:17,609 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d 2024-11-23T18:51:17,612 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/zookeeper_0, clientPort=61095, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:51:17,613 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61095 2024-11-23T18:51:17,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:51:17,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:51:17,627 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f with version=8 2024-11-23T18:51:17,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase-staging 2024-11-23T18:51:17,630 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:51:17,630 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:51:17,631 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36073 2024-11-23T18:51:17,633 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36073 connecting to ZooKeeper ensemble=127.0.0.1:61095 2024-11-23T18:51:17,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360730x0, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:51:17,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36073-0x10169092e970000 connected 2024-11-23T18:51:17,781 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:17,786 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:17,798 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,800 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:17,802 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f, hbase.cluster.distributed=false 2024-11-23T18:51:17,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:51:17,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36073 2024-11-23T18:51:17,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36073 2024-11-23T18:51:17,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36073 2024-11-23T18:51:17,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36073 2024-11-23T18:51:17,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36073 2024-11-23T18:51:17,820 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:51:17,820 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:51:17,821 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:51:17,821 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34197 2024-11-23T18:51:17,823 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34197 connecting to ZooKeeper ensemble=127.0.0.1:61095 2024-11-23T18:51:17,823 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,825 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341970x0, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:51:17,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341970x0, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:17,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34197-0x10169092e970001 connected 2024-11-23T18:51:17,841 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:51:17,842 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:51:17,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:51:17,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:51:17,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34197 2024-11-23T18:51:17,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34197 2024-11-23T18:51:17,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34197 2024-11-23T18:51:17,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34197 2024-11-23T18:51:17,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34197 2024-11-23T18:51:17,856 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:36073 2024-11-23T18:51:17,857 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,36073,1732387877629 2024-11-23T18:51:17,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:17,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:17,862 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,36073,1732387877629 2024-11-23T18:51:17,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:51:17,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:17,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:17,872 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:51:17,873 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,36073,1732387877629 from backup master directory 2024-11-23T18:51:17,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,36073,1732387877629 2024-11-23T18:51:17,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:17,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:17,882 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:51:17,882 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,36073,1732387877629 2024-11-23T18:51:17,889 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/hbase.id] with ID: 84b444aa-bf0e-4453-a104-1750afcbafe8 2024-11-23T18:51:17,889 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/.tmp/hbase.id 2024-11-23T18:51:17,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:51:17,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:51:17,898 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/.tmp/hbase.id]:[hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/hbase.id] 2024-11-23T18:51:17,916 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:17,917 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:51:17,919 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T18:51:17,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:17,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:51:17,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:51:18,306 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:51:18,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:18,329 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:18,334 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:51:18,335 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:51:18,336 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:51:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:51:18,347 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store 2024-11-23T18:51:18,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:51:18,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:51:18,355 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:18,355 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:18,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387878355Disabling compacts and flushes for region at 1732387878355Disabling writes for close at 1732387878355Writing region close event to WAL at 1732387878355Closed at 1732387878355 2024-11-23T18:51:18,356 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/.initializing 2024-11-23T18:51:18,356 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629 2024-11-23T18:51:18,357 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:51:18,360 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C36073%2C1732387877629, suffix=, logDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629, archiveDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/oldWALs, maxLogs=10 2024-11-23T18:51:18,371 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360, exclude list is [], retry=0 2024-11-23T18:51:18,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45909,DS-ff9df5de-0527-4946-b726-e4069b4967c4,DISK] 2024-11-23T18:51:18,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44293,DS-23f485e2-c948-49f1-beb5-7857f0b7485b,DISK] 2024-11-23T18:51:18,377 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 2024-11-23T18:51:18,377 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46267:46267),(127.0.0.1/127.0.0.1:35143:35143)] 2024-11-23T18:51:18,377 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:18,378 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:18,378 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,378 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:51:18,382 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:18,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,385 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:51:18,385 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,385 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:18,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:51:18,388 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:18,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:51:18,391 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:18,392 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,392 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,393 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,395 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,395 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,395 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:51:18,397 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:18,400 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:18,400 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868357, jitterRate=0.1041741669178009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:51:18,401 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387878378Initializing all the Stores at 1732387878379 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387878379Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387878380 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387878380Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387878380Cleaning up temporary data from old regions at 1732387878395 (+15 ms)Region opened successfully at 1732387878401 (+6 ms) 2024-11-23T18:51:18,402 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:51:18,406 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ee1562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:51:18,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:51:18,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:51:18,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:51:18,407 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:51:18,408 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T18:51:18,409 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T18:51:18,409 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:51:18,411 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:51:18,412 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:51:18,451 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:51:18,451 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:51:18,452 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:51:18,461 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:51:18,462 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:51:18,463 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:51:18,472 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:51:18,473 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:51:18,482 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:51:18,485 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:51:18,493 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:51:18,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:18,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:18,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,504 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,36073,1732387877629, sessionid=0x10169092e970000, setting cluster-up flag (Was=false) 2024-11-23T18:51:18,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,556 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:51:18,558 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,36073,1732387877629 2024-11-23T18:51:18,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:18,609 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:51:18,613 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,36073,1732387877629 2024-11-23T18:51:18,616 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:51:18,620 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:18,621 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:51:18,621 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:51:18,622 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,36073,1732387877629 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:51:18,624 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:18,624 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:18,624 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:18,624 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:18,624 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:51:18,625 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,625 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:51:18,625 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387908626 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:51:18,626 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:51:18,627 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:18,627 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:51:18,627 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,628 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:51:18,628 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:51:18,628 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:51:18,628 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:51:18,628 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:51:18,629 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,629 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387878629,5,FailOnTimeoutGroup] 2024-11-23T18:51:18,629 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:51:18,640 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387878629,5,FailOnTimeoutGroup] 2024-11-23T18:51:18,640 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,640 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:51:18,640 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,640 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:51:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:51:18,648 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:51:18,648 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(746): ClusterId : 84b444aa-bf0e-4453-a104-1750afcbafe8 2024-11-23T18:51:18,648 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:51:18,648 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f 2024-11-23T18:51:18,652 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:51:18,652 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:51:18,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:51:18,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:51:18,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:18,662 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:51:18,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:51:18,663 DEBUG [RS:0;2eda60943ea1:34197 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45abe08f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:51:18,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:51:18,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:18,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:51:18,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:51:18,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:18,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:51:18,669 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:51:18,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:18,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:51:18,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:51:18,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:18,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:18,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:51:18,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740 2024-11-23T18:51:18,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740 2024-11-23T18:51:18,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:51:18,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:51:18,676 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:51:18,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:51:18,680 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:18,681 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=827731, jitterRate=0.052514418959617615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:51:18,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387878660Initializing all the Stores at 1732387878662 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387878662Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387878662Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387878662Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387878662Cleaning up temporary data from old regions at 1732387878676 (+14 ms)Region opened successfully at 1732387878681 (+5 ms) 2024-11-23T18:51:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:51:18,682 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:51:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:51:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:51:18,682 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:34197 2024-11-23T18:51:18,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:51:18,682 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:51:18,682 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:51:18,682 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:51:18,682 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:18,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387878682Disabling compacts and flushes for region at 1732387878682Disabling writes for close at 1732387878682Writing region close event to WAL at 1732387878682Closed at 1732387878682 2024-11-23T18:51:18,683 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,36073,1732387877629 with port=34197, startcode=1732387877820 2024-11-23T18:51:18,683 DEBUG [RS:0;2eda60943ea1:34197 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:51:18,685 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:18,685 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:51:18,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:51:18,686 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60241, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:51:18,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36073 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36073 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,687 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:51:18,688 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f 2024-11-23T18:51:18,688 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38745 2024-11-23T18:51:18,688 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:51:18,689 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:51:18,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:51:18,693 DEBUG [RS:0;2eda60943ea1:34197 {}] zookeeper.ZKUtil(111): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,693 WARN [RS:0;2eda60943ea1:34197 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:51:18,694 INFO [RS:0;2eda60943ea1:34197 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:18,694 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,694 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,34197,1732387877820] 2024-11-23T18:51:18,698 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:51:18,700 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:51:18,700 INFO [RS:0;2eda60943ea1:34197 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:51:18,700 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,704 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:51:18,705 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:51:18,705 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,705 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:51:18,706 DEBUG [RS:0;2eda60943ea1:34197 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,707 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34197,1732387877820-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:51:18,721 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:51:18,722 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34197,1732387877820-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,722 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,722 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.Replication(171): 2eda60943ea1,34197,1732387877820 started 2024-11-23T18:51:18,735 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:18,736 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,34197,1732387877820, RpcServer on 2eda60943ea1/172.17.0.2:34197, sessionid=0x10169092e970001 2024-11-23T18:51:18,736 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:51:18,736 DEBUG [RS:0;2eda60943ea1:34197 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,736 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,34197,1732387877820' 2024-11-23T18:51:18,736 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,34197,1732387877820 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,34197,1732387877820' 2024-11-23T18:51:18,737 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:51:18,738 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:51:18,738 DEBUG [RS:0;2eda60943ea1:34197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:51:18,738 INFO [RS:0;2eda60943ea1:34197 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:51:18,738 INFO [RS:0;2eda60943ea1:34197 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:51:18,839 WARN [2eda60943ea1:36073 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:51:18,839 INFO [RS:0;2eda60943ea1:34197 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:51:18,842 INFO [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C34197%2C1732387877820, suffix=, logDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820, archiveDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/oldWALs, maxLogs=32 2024-11-23T18:51:18,863 DEBUG [RS:0;2eda60943ea1:34197 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846, exclude list is [], retry=0 2024-11-23T18:51:18,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45909,DS-ff9df5de-0527-4946-b726-e4069b4967c4,DISK] 2024-11-23T18:51:18,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44293,DS-23f485e2-c948-49f1-beb5-7857f0b7485b,DISK] 2024-11-23T18:51:18,869 INFO [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 2024-11-23T18:51:18,870 DEBUG [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46267:46267),(127.0.0.1/127.0.0.1:35143:35143)] 2024-11-23T18:51:19,089 DEBUG [2eda60943ea1:36073 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T18:51:19,091 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:19,096 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,34197,1732387877820, state=OPENING 2024-11-23T18:51:19,135 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:51:19,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:19,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:19,147 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:51:19,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:19,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:19,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,34197,1732387877820}] 2024-11-23T18:51:19,304 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:51:19,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:51:19,310 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:51:19,310 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:19,311 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:51:19,315 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C34197%2C1732387877820.meta, suffix=.meta, logDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820, archiveDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/oldWALs, maxLogs=32 2024-11-23T18:51:19,330 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta, exclude list is [], retry=0 2024-11-23T18:51:19,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45909,DS-ff9df5de-0527-4946-b726-e4069b4967c4,DISK] 2024-11-23T18:51:19,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44293,DS-23f485e2-c948-49f1-beb5-7857f0b7485b,DISK] 2024-11-23T18:51:19,354 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta 2024-11-23T18:51:19,354 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35143:35143),(127.0.0.1/127.0.0.1:46267:46267)] 2024-11-23T18:51:19,355 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:19,355 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:51:19,355 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:51:19,355 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:51:19,355 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:51:19,356 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:19,356 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:51:19,356 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:51:19,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:51:19,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:51:19,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:19,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:19,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:51:19,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:51:19,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:19,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:19,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:51:19,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:51:19,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:19,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:19,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:51:19,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:51:19,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:19,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:19,367 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:51:19,368 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740 2024-11-23T18:51:19,370 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740 2024-11-23T18:51:19,372 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:51:19,372 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:51:19,372 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:51:19,374 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:51:19,375 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771472, jitterRate=-0.019023388624191284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:51:19,376 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:51:19,377 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387879356Writing region info on filesystem at 1732387879356Initializing all the Stores at 1732387879358 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387879358Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387879358Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387879358Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387879358Cleaning up temporary data from old regions at 1732387879372 (+14 ms)Running coprocessor post-open hooks at 1732387879376 (+4 ms)Region opened successfully at 1732387879377 (+1 ms) 2024-11-23T18:51:19,378 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387879303 2024-11-23T18:51:19,381 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:51:19,382 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:51:19,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:19,385 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,34197,1732387877820, state=OPEN 2024-11-23T18:51:19,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:51:19,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:51:19,427 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:19,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:19,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:19,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:51:19,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,34197,1732387877820 in 279 msec 2024-11-23T18:51:19,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:51:19,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 747 msec 2024-11-23T18:51:19,437 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:19,437 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:51:19,438 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:51:19,438 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,34197,1732387877820, seqNum=-1] 2024-11-23T18:51:19,439 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:51:19,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47567, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:51:19,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 828 msec 2024-11-23T18:51:19,447 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387879447, completionTime=-1 2024-11-23T18:51:19,447 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T18:51:19,447 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:51:19,449 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T18:51:19,449 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732387939449 2024-11-23T18:51:19,449 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732387999449 2024-11-23T18:51:19,449 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-23T18:51:19,450 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,450 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,450 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,450 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:36073, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,450 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,451 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,452 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:51:19,455 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.572sec 2024-11-23T18:51:19,455 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:51:19,455 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:51:19,455 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:51:19,455 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:51:19,456 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:51:19,456 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:51:19,456 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:51:19,459 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:51:19,459 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:51:19,459 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36073,1732387877629-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:19,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cafcc38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:19,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,36073,-1 for getting cluster id 2024-11-23T18:51:19,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:51:19,551 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84b444aa-bf0e-4453-a104-1750afcbafe8' 2024-11-23T18:51:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:51:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84b444aa-bf0e-4453-a104-1750afcbafe8" 2024-11-23T18:51:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e84f11b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,36073,-1] 2024-11-23T18:51:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:51:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:19,554 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:51:19,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a77a0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:19,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:51:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,34197,1732387877820, seqNum=-1] 2024-11-23T18:51:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:51:19,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:51:19,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,36073,1732387877629 2024-11-23T18:51:19,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:19,565 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-23T18:51:19,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:19,581 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:19,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:19,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:19,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:19,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ee815b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:19,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a1625d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:19,676 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2874572a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-35751-hadoop-hdfs-3_4_1-tests_jar-_-any-1732885990366636879/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:19,676 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8b61256{HTTP/1.1, (http/1.1)}{localhost:35751} 2024-11-23T18:51:19,676 INFO [Time-limited test {}] server.Server(415): Started @73826ms 2024-11-23T18:51:19,678 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:19,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:19,725 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:19,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:19,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:19,728 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:51:19,728 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2037e379{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:19,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71aa8073{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:19,825 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ea0ce88{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-35023-hadoop-hdfs-3_4_1-tests_jar-_-any-7637043853178672398/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:19,825 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27724107{HTTP/1.1, (http/1.1)}{localhost:35023} 2024-11-23T18:51:19,825 INFO [Time-limited test {}] server.Server(415): Started @73975ms 2024-11-23T18:51:19,827 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:19,856 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:19,861 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:19,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:19,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:19,862 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:19,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34d98298{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:19,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28ab81a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:19,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@245d91f8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-34585-hadoop-hdfs-3_4_1-tests_jar-_-any-1951387136096953836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:19,959 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3bc43b80{HTTP/1.1, (http/1.1)}{localhost:34585} 2024-11-23T18:51:19,959 INFO [Time-limited test {}] server.Server(415): Started @74109ms 2024-11-23T18:51:19,962 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:21,262 WARN [Thread-548 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data5/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,262 WARN [Thread-549 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data6/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,284 WARN [Thread-490 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5ee52793d61a004f with lease ID 0xd33df1f093b822b8: Processing first storage report for DS-e58e06f7-3423-4416-8e00-3d16d00e31ec from datanode DatanodeRegistration(127.0.0.1:34391, datanodeUuid=d5a133dd-fbd4-46d8-a038-7fd4418d0a06, infoPort=35703, infoSecurePort=0, ipcPort=38191, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ee52793d61a004f with lease ID 0xd33df1f093b822b8: from storage DS-e58e06f7-3423-4416-8e00-3d16d00e31ec node DatanodeRegistration(127.0.0.1:34391, datanodeUuid=d5a133dd-fbd4-46d8-a038-7fd4418d0a06, infoPort=35703, infoSecurePort=0, ipcPort=38191, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5ee52793d61a004f with lease ID 0xd33df1f093b822b8: Processing first storage report for DS-a39ebddb-13a6-48be-b319-6bf32edcfddc from datanode DatanodeRegistration(127.0.0.1:34391, datanodeUuid=d5a133dd-fbd4-46d8-a038-7fd4418d0a06, infoPort=35703, infoSecurePort=0, ipcPort=38191, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ee52793d61a004f with lease ID 0xd33df1f093b822b8: from storage DS-a39ebddb-13a6-48be-b319-6bf32edcfddc node DatanodeRegistration(127.0.0.1:34391, datanodeUuid=d5a133dd-fbd4-46d8-a038-7fd4418d0a06, infoPort=35703, infoSecurePort=0, ipcPort=38191, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,635 WARN [Thread-560 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data8/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,635 WARN [Thread-559 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data7/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,653 WARN [Thread-512 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36e749eb22870d8f with lease ID 0xd33df1f093b822b9: Processing first storage report for DS-93871931-11b0-4641-89ca-81f04a481170 from datanode DatanodeRegistration(127.0.0.1:36063, datanodeUuid=b600b879-1003-484c-8323-7488f2a57195, infoPort=38679, infoSecurePort=0, ipcPort=46063, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36e749eb22870d8f with lease ID 0xd33df1f093b822b9: from storage DS-93871931-11b0-4641-89ca-81f04a481170 node DatanodeRegistration(127.0.0.1:36063, datanodeUuid=b600b879-1003-484c-8323-7488f2a57195, infoPort=38679, infoSecurePort=0, ipcPort=46063, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36e749eb22870d8f with lease ID 0xd33df1f093b822b9: Processing first storage report for DS-4cd768aa-a0b9-4526-8de8-d266b304065d from datanode DatanodeRegistration(127.0.0.1:36063, datanodeUuid=b600b879-1003-484c-8323-7488f2a57195, infoPort=38679, infoSecurePort=0, ipcPort=46063, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36e749eb22870d8f with lease ID 0xd33df1f093b822b9: from storage DS-4cd768aa-a0b9-4526-8de8-d266b304065d node DatanodeRegistration(127.0.0.1:36063, datanodeUuid=b600b879-1003-484c-8323-7488f2a57195, infoPort=38679, infoSecurePort=0, ipcPort=46063, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,708 WARN [Thread-570 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data9/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,708 WARN [Thread-571 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data10/current/BP-1056060911-172.17.0.2-1732387873601/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:21,730 WARN [Thread-534 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:21,732 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x657bee140ef9781a with lease ID 0xd33df1f093b822ba: Processing first storage report for DS-1a8c735b-b358-4e37-8c74-f4a0b1c6bd3d from datanode DatanodeRegistration(127.0.0.1:44603, datanodeUuid=3a531745-f7a0-4e15-a187-c12e49aa3c8c, infoPort=45593, infoSecurePort=0, ipcPort=34493, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x657bee140ef9781a with lease ID 0xd33df1f093b822ba: from storage DS-1a8c735b-b358-4e37-8c74-f4a0b1c6bd3d node DatanodeRegistration(127.0.0.1:44603, datanodeUuid=3a531745-f7a0-4e15-a187-c12e49aa3c8c, infoPort=45593, infoSecurePort=0, ipcPort=34493, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x657bee140ef9781a with lease ID 0xd33df1f093b822ba: Processing first storage report for DS-31797e69-cd9e-4424-b6cc-e50a8323ca5e from datanode DatanodeRegistration(127.0.0.1:44603, datanodeUuid=3a531745-f7a0-4e15-a187-c12e49aa3c8c, infoPort=45593, infoSecurePort=0, ipcPort=34493, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601) 2024-11-23T18:51:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x657bee140ef9781a with lease ID 0xd33df1f093b822ba: from storage DS-31797e69-cd9e-4424-b6cc-e50a8323ca5e node DatanodeRegistration(127.0.0.1:44603, datanodeUuid=3a531745-f7a0-4e15-a187-c12e49aa3c8c, infoPort=45593, infoSecurePort=0, ipcPort=34493, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:21,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:51:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,36073,1732387877629 2024-11-23T18:51:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4acd79ee 2024-11-23T18:51:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:51:21,814 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:51:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-23T18:51:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-23T18:51:21,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:51:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-23T18:51:21,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:51:21,820 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:21,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-23T18:51:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:51:21,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:51:21,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741835_1011 (size=393) 2024-11-23T18:51:21,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741835_1011 (size=393) 2024-11-23T18:51:21,833 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ed2b33bd370312ad2d91f62b9ac1b24, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f 2024-11-23T18:51:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741836_1012 (size=76) 2024-11-23T18:51:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45909 is added to blk_1073741836_1012 (size=76) 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 0ed2b33bd370312ad2d91f62b9ac1b24, disabling compactions & flushes 2024-11-23T18:51:21,841 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. after waiting 0 ms 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:21,841 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:21,841 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ed2b33bd370312ad2d91f62b9ac1b24: Waiting for close lock at 1732387881841Disabling compacts and flushes for region at 1732387881841Disabling writes for close at 1732387881841Writing region close event to WAL at 1732387881841Closed at 1732387881841 2024-11-23T18:51:21,843 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:51:21,843 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732387881843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387881843"}]},"ts":"1732387881843"} 2024-11-23T18:51:21,845 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:51:21,847 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:51:21,847 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387881847"}]},"ts":"1732387881847"} 2024-11-23T18:51:21,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-23T18:51:21,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ed2b33bd370312ad2d91f62b9ac1b24, ASSIGN}] 2024-11-23T18:51:21,851 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ed2b33bd370312ad2d91f62b9ac1b24, ASSIGN 2024-11-23T18:51:21,853 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ed2b33bd370312ad2d91f62b9ac1b24, ASSIGN; state=OFFLINE, location=2eda60943ea1,34197,1732387877820; forceNewPlan=false, retain=false 2024-11-23T18:51:22,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ed2b33bd370312ad2d91f62b9ac1b24, regionState=OPENING, regionLocation=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:22,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ed2b33bd370312ad2d91f62b9ac1b24, ASSIGN because future has completed 2024-11-23T18:51:22,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed2b33bd370312ad2d91f62b9ac1b24, server=2eda60943ea1,34197,1732387877820}] 2024-11-23T18:51:22,173 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:22,173 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ed2b33bd370312ad2d91f62b9ac1b24, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:22,173 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,173 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:22,174 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,174 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,176 INFO [StoreOpener-0ed2b33bd370312ad2d91f62b9ac1b24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,178 INFO [StoreOpener-0ed2b33bd370312ad2d91f62b9ac1b24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed2b33bd370312ad2d91f62b9ac1b24 columnFamilyName info 2024-11-23T18:51:22,178 DEBUG [StoreOpener-0ed2b33bd370312ad2d91f62b9ac1b24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:22,178 INFO [StoreOpener-0ed2b33bd370312ad2d91f62b9ac1b24-1 {}] regionserver.HStore(327): Store=0ed2b33bd370312ad2d91f62b9ac1b24/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:22,179 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,180 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,180 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,181 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,181 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,183 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,186 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:22,186 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0ed2b33bd370312ad2d91f62b9ac1b24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=726482, jitterRate=-0.07623091340065002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:51:22,186 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:22,187 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0ed2b33bd370312ad2d91f62b9ac1b24: Running coprocessor pre-open hook at 1732387882174Writing region info on filesystem at 1732387882174Initializing all the Stores at 1732387882175 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387882175Cleaning up temporary data from old regions at 1732387882181 (+6 ms)Running coprocessor post-open hooks at 1732387882186 (+5 ms)Region opened successfully at 1732387882187 (+1 ms) 2024-11-23T18:51:22,189 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24., pid=6, masterSystemTime=1732387882162 2024-11-23T18:51:22,191 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:22,191 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:22,192 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ed2b33bd370312ad2d91f62b9ac1b24, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,34197,1732387877820 2024-11-23T18:51:22,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed2b33bd370312ad2d91f62b9ac1b24, server=2eda60943ea1,34197,1732387877820 because future has completed 2024-11-23T18:51:22,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:51:22,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ed2b33bd370312ad2d91f62b9ac1b24, server=2eda60943ea1,34197,1732387877820 in 188 msec 2024-11-23T18:51:22,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:51:22,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ed2b33bd370312ad2d91f62b9ac1b24, ASSIGN in 350 msec 2024-11-23T18:51:22,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:51:22,203 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387882203"}]},"ts":"1732387882203"} 2024-11-23T18:51:22,206 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-23T18:51:22,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:51:22,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 391 msec 2024-11-23T18:51:22,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-23T18:51:22,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-23T18:51:22,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-23T18:51:24,865 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:51:24,869 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:24,895 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:24,904 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:51:24,905 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-23T18:51:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36073 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:51:31,915 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-23T18:51:31,916 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestLogRolling-testLogRollOnDatanodeDeath get assigned. Timeout = 60000ms 2024-11-23T18:51:31,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:51:31,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestLogRolling-testLogRollOnDatanodeDeath assigned to meta. Checking AM states. 2024-11-23T18:51:31,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:51:31,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestLogRolling-testLogRollOnDatanodeDeath assigned. 2024-11-23T18:51:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24., hostname=2eda60943ea1,34197,1732387877820, seqNum=2] 2024-11-23T18:51:31,934 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-23T18:51:31,937 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-23T18:51:31,937 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1198281028_22 at /127.0.0.1:37390 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37390 dst: /127.0.0.1:44293 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1417471866_22 at /127.0.0.1:37426 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37426 dst: /127.0.0.1:44293 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1417471866_22 at /127.0.0.1:36060 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45909:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36060 dst: /127.0.0.1:45909 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1417471866_22 at /127.0.0.1:36048 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45909:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36048 dst: /127.0.0.1:45909 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1417471866_22 at /127.0.0.1:37430 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37430 dst: /127.0.0.1:44293 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:31,939 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1198281028_22 at /127.0.0.1:36012 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45909:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36012 dst: /127.0.0.1:45909 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ff9dca{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:32,065 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c6b6815{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:32,065 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:32,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52967733{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:32,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6e73dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:32,068 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:32,068 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid 01d6c3f6-3515-48a5-9cd0-b8ec8155928e) service to localhost/127.0.0.1:38745 2024-11-23T18:51:32,068 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:32,069 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:32,070 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data1/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:32,070 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data2/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:32,071 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:32,080 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:32,083 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:32,084 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:32,084 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:32,084 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:32,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f102659{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:32,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dbae964{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:32,179 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7aba7c39{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/java.io.tmpdir/jetty-localhost-44515-hadoop-hdfs-3_4_1-tests_jar-_-any-5143395320113986393/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:32,179 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@387fbdda{HTTP/1.1, (http/1.1)}{localhost:44515} 2024-11-23T18:51:32,179 INFO [Time-limited test {}] server.Server(415): Started @86329ms 2024-11-23T18:51:32,181 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:32,198 WARN [AsyncFSWAL-0-hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f-prefix:2eda60943ea1,34197,1732387877820 {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,198 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2eda60943ea1%2C34197%2C1732387877820:(num 1732387878846) roll requested 2024-11-23T18:51:32,219 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387892199, exclude list is [], retry=0 2024-11-23T18:51:32,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44293,DS-23f485e2-c948-49f1-beb5-7857f0b7485b,DISK] 2024-11-23T18:51:32,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-93871931-11b0-4641-89ca-81f04a481170,DISK] 2024-11-23T18:51:32,225 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 with entries=3, filesize=2.68 KB; new WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387892199 2024-11-23T18:51:32,226 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35143:35143),(127.0.0.1/127.0.0.1:38679:38679)] 2024-11-23T18:51:32,226 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 is not closed yet, will try archiving it next time 2024-11-23T18:51:32,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:51:32,227 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:51:32,227 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:32,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:32,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:32,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:51:32,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1817200040, stopped=false 2024-11-23T18:51:32,228 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,36073,1732387877629 2024-11-23T18:51:32,228 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-23T18:51:32,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:51:32,228 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-23T18:51:32,228 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 2024-11-23T18:51:32,231 WARN [IPC Server handler 0 on default port 38745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 has not been closed. Lease recovery is in progress. RecoveryId = 1014 for block blk_1073741833_1009 2024-11-23T18:51:32,234 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 after 4ms 2024-11-23T18:51:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:32,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:51:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:32,282 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:51:32,282 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:32,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:32,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:32,283 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,34197,1732387877820' ***** 2024-11-23T18:51:32,283 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:51:32,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:32,283 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:51:32,283 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:51:32,283 INFO [RS:0;2eda60943ea1:34197 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:51:32,284 INFO [RS:0;2eda60943ea1:34197 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:51:32,284 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ed2b33bd370312ad2d91f62b9ac1b24 2024-11-23T18:51:32,284 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,34197,1732387877820 2024-11-23T18:51:32,284 INFO [RS:0;2eda60943ea1:34197 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:51:32,284 INFO [RS:0;2eda60943ea1:34197 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:34197. 2024-11-23T18:51:32,284 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ed2b33bd370312ad2d91f62b9ac1b24, disabling compactions & flushes 2024-11-23T18:51:32,284 DEBUG [RS:0;2eda60943ea1:34197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:51:32,284 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:32,284 DEBUG [RS:0;2eda60943ea1:34197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:32,284 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:32,285 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. after waiting 0 ms 2024-11-23T18:51:32,285 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:51:32,285 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:32,285 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:51:32,285 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:51:32,285 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:51:32,285 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ed2b33bd370312ad2d91f62b9ac1b24 1/1 column families, dataSize=2.05 KB heapSize=2.45 KB 2024-11-23T18:51:32,285 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:51:32,285 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1325): Online Regions={0ed2b33bd370312ad2d91f62b9ac1b24=TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24., 1588230740=hbase:meta,,1.1588230740} 2024-11-23T18:51:32,285 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1351): Waiting on 0ed2b33bd370312ad2d91f62b9ac1b24, 1588230740 2024-11-23T18:51:32,285 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:51:32,285 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:51:32,285 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:51:32,286 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:51:32,286 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:51:32,286 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-23T18:51:32,287 WARN [AsyncFSWAL-0-hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f-prefix:2eda60943ea1,34197,1732387877820.meta {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,287 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2eda60943ea1%2C34197%2C1732387877820.meta:.meta(num 1732387879318) roll requested 2024-11-23T18:51:32,302 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387892287.meta, exclude list is [], retry=0 2024-11-23T18:51:32,303 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/.tmp/info/1b1579f913f343318f53f1661977bae4 is 1055, key is row0001/info:/1732387891932/Put/seqid=0 2024-11-23T18:51:32,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-93871931-11b0-4641-89ca-81f04a481170,DISK] 2024-11-23T18:51:32,305 INFO [regionserver/2eda60943ea1:0.logRoller {}] monitor.ExcludeDatanodeManager(82): Added datanode: DatanodeInfoWithStorage[127.0.0.1:45909,DS-ff9df5de-0527-4946-b726-e4069b4967c4,DISK] to exclude cache by [connect error] success, current excludeDNsCache size=1 2024-11-23T18:51:32,305 WARN [regionserver/2eda60943ea1:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(578): create fan-out dfs output /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387892287.meta failed, retry = 0 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: /127.0.0.1:45909 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741839_1016 (size=7062) 2024-11-23T18:51:32,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741839_1016 (size=7062) 2024-11-23T18:51:32,309 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.05 KB at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/.tmp/info/1b1579f913f343318f53f1661977bae4 2024-11-23T18:51:32,319 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/.tmp/info/1b1579f913f343318f53f1661977bae4 as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/info/1b1579f913f343318f53f1661977bae4 2024-11-23T18:51:32,327 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/info/1b1579f913f343318f53f1661977bae4, entries=2, sequenceid=6, filesize=6.9 K 2024-11-23T18:51:32,328 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.05 KB/2102, heapSize ~2.44 KB/2496, currentSize=0 B/0 for 0ed2b33bd370312ad2d91f62b9ac1b24 in 43ms, sequenceid=6, compaction requested=false 2024-11-23T18:51:32,333 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ed2b33bd370312ad2d91f62b9ac1b24/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T18:51:32,334 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:32,334 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ed2b33bd370312ad2d91f62b9ac1b24: Waiting for close lock at 1732387892284Running coprocessor pre-close hooks at 1732387892284Disabling compacts and flushes for region at 1732387892284Disabling writes for close at 1732387892285 (+1 ms)Obtaining lock to block concurrent updates at 1732387892285Preparing flush snapshotting stores in 0ed2b33bd370312ad2d91f62b9ac1b24 at 1732387892285Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24., syncing WAL and waiting on mvcc, flushsize=dataSize=2102, getHeapSize=2496, getOffHeapSize=0, getCellsCount=2 at 1732387892285Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. at 1732387892286 (+1 ms)Flushing 0ed2b33bd370312ad2d91f62b9ac1b24/info: creating writer at 1732387892286Flushing 0ed2b33bd370312ad2d91f62b9ac1b24/info: appending metadata at 1732387892302 (+16 ms)Flushing 0ed2b33bd370312ad2d91f62b9ac1b24/info: closing flushed file at 1732387892302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30735b71: reopening flushed file at 1732387892318 (+16 ms)Finished flush of dataSize ~2.05 KB/2102, heapSize ~2.44 KB/2496, currentSize=0 B/0 for 0ed2b33bd370312ad2d91f62b9ac1b24 in 43ms, sequenceid=6, compaction requested=false at 1732387892328 (+10 ms)Writing region close event to WAL at 1732387892329 (+1 ms)Running coprocessor post-close hooks at 1732387892334 (+5 ms)Closed at 1732387892334 2024-11-23T18:51:32,334 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24. 2024-11-23T18:51:32,406 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387892287.meta, exclude list is [(127.0.0.1/127.0.0.1:46267:46267)], retry=1 2024-11-23T18:51:32,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1417471866_22 at /127.0.0.1:39334 [Receiving block BP-1056060911-172.17.0.2-1732387873601:blk_1073741838_1015] {}] datanode.DataXceiver(331): 127.0.0.1:36063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39334 dst: /127.0.0.1:36063 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-93871931-11b0-4641-89ca-81f04a481170,DISK] 2024-11-23T18:51:32,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44603,DS-1a8c735b-b358-4e37-8c74-f4a0b1c6bd3d,DISK] 2024-11-23T18:51:32,412 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta with entries=9, filesize=2.52 KB; new WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387892287.meta 2024-11-23T18:51:32,413 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38679:38679),(127.0.0.1/127.0.0.1:45593:45593)] 2024-11-23T18:51:32,413 DEBUG [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta is not closed yet, will try archiving it next time 2024-11-23T18:51:32,413 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,413 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:32,414 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta 2024-11-23T18:51:32,414 WARN [IPC Server handler 2 on default port 38745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta has not been closed. Lease recovery is in progress. RecoveryId = 1018 for block blk_1073741834_1010 2024-11-23T18:51:32,415 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta after 1ms 2024-11-23T18:51:32,430 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/info/df412a20ecd444c6893a786151f05ff5 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732387881815.0ed2b33bd370312ad2d91f62b9ac1b24./info:regioninfo/1732387882192/Put/seqid=0 2024-11-23T18:51:32,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741841_1019 (size=7089) 2024-11-23T18:51:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741841_1019 (size=7089) 2024-11-23T18:51:32,437 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/info/df412a20ecd444c6893a786151f05ff5 2024-11-23T18:51:32,460 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/ns/08555ecb33c843b4a274d8c68b1d872c is 43, key is default/ns:d/1732387879441/Put/seqid=0 2024-11-23T18:51:32,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-23T18:51:32,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34391 is added to blk_1073741842_1020 (size=5153) 2024-11-23T18:51:32,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741842_1020 (size=5153) 2024-11-23T18:51:32,467 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/ns/08555ecb33c843b4a274d8c68b1d872c 2024-11-23T18:51:32,485 DEBUG [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T18:51:32,490 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/table/8c0018f6b5cd41dea1fe59c3fb1498ca is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732387882203/Put/seqid=0 2024-11-23T18:51:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741843_1021 (size=5424) 2024-11-23T18:51:32,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741843_1021 (size=5424) 2024-11-23T18:51:32,496 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/table/8c0018f6b5cd41dea1fe59c3fb1498ca 2024-11-23T18:51:32,504 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/info/df412a20ecd444c6893a786151f05ff5 as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/info/df412a20ecd444c6893a786151f05ff5 2024-11-23T18:51:32,512 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/info/df412a20ecd444c6893a786151f05ff5, entries=10, sequenceid=11, filesize=6.9 K 2024-11-23T18:51:32,513 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/ns/08555ecb33c843b4a274d8c68b1d872c as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/ns/08555ecb33c843b4a274d8c68b1d872c 2024-11-23T18:51:32,521 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/ns/08555ecb33c843b4a274d8c68b1d872c, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T18:51:32,522 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/.tmp/table/8c0018f6b5cd41dea1fe59c3fb1498ca as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/table/8c0018f6b5cd41dea1fe59c3fb1498ca 2024-11-23T18:51:32,530 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/table/8c0018f6b5cd41dea1fe59c3fb1498ca, entries=2, sequenceid=11, filesize=5.3 K 2024-11-23T18:51:32,532 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 246ms, sequenceid=11, compaction requested=false 2024-11-23T18:51:32,538 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T18:51:32,539 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:51:32,539 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:32,539 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387892285Running coprocessor pre-close hooks at 1732387892285Disabling compacts and flushes for region at 1732387892285Disabling writes for close at 1732387892286 (+1 ms)Obtaining lock to block concurrent updates at 1732387892286Preparing flush snapshotting stores in 1588230740 at 1732387892286Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732387892286Flushing stores of hbase:meta,,1.1588230740 at 1732387892413 (+127 ms)Flushing 1588230740/info: creating writer at 1732387892414 (+1 ms)Flushing 1588230740/info: appending metadata at 1732387892429 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732387892429Flushing 1588230740/ns: creating writer at 1732387892443 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732387892459 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732387892460 (+1 ms)Flushing 1588230740/table: creating writer at 1732387892474 (+14 ms)Flushing 1588230740/table: appending metadata at 1732387892489 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732387892489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4688dedb: reopening flushed file at 1732387892503 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39bd98fd: reopening flushed file at 1732387892512 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e1a57be: reopening flushed file at 1732387892521 (+9 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 246ms, sequenceid=11, compaction requested=false at 1732387892532 (+11 ms)Writing region close event to WAL at 1732387892533 (+1 ms)Running coprocessor post-close hooks at 1732387892539 (+6 ms)Closed at 1732387892539 2024-11-23T18:51:32,539 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:32,636 WARN [Thread-609 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5489764f65d35ffe with lease ID 0xd33df1f093b822bb: from storage DS-ff9df5de-0527-4946-b726-e4069b4967c4 node DatanodeRegistration(127.0.0.1:39117, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=39817, infoSecurePort=0, ipcPort=33577, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5489764f65d35ffe with lease ID 0xd33df1f093b822bb: from storage DS-f6e7b460-6311-41ac-8a1a-0d7de90ffc65 node DatanodeRegistration(127.0.0.1:39117, datanodeUuid=01d6c3f6-3515-48a5-9cd0-b8ec8155928e, infoPort=39817, infoSecurePort=0, ipcPort=33577, storageInfo=lv=-57;cid=testClusterID;nsid=973898583;c=1732387873601), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:32,686 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,34197,1732387877820; all regions closed. 2024-11-23T18:51:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741840_1017 (size=825) 2024-11-23T18:51:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741840_1017 (size=825) 2024-11-23T18:51:32,708 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:51:32,712 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T18:51:32,712 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T18:51:36,237 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 after 4008ms 2024-11-23T18:51:36,417 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta after 4003ms 2024-11-23T18:51:37,335 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:37,352 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:37,352 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:37,687 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.async.wait.on.shutdown.seconds" 2024-11-23T18:51:37,698 DEBUG [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/oldWALs 2024-11-23T18:51:37,698 INFO [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C34197%2C1732387877820.meta:.meta(num 1732387892287) 2024-11-23T18:51:37,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741837_1013 (size=2288) 2024-11-23T18:51:37,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741837_1013 (size=2288) 2024-11-23T18:51:37,861 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:51:37,866 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:37,883 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:37,884 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:38,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:39,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:40,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:41,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:42,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor108.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:42,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:51:42,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:51:42,622 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@14f1938f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1056060911-172.17.0.2-1732387873601:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:45909,null,null]) java.net.ConnectException: Call From 2eda60943ea1/172.17.0.2 to localhost:38649 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-23T18:51:42,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741833_1014 (size=1599) 2024-11-23T18:51:42,700 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.async.wait.on.shutdown.seconds" 2024-11-23T18:51:42,708 DEBUG [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/oldWALs 2024-11-23T18:51:42,708 INFO [RS:0;2eda60943ea1:34197 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C34197%2C1732387877820:(num 1732387892199) 2024-11-23T18:51:42,708 DEBUG [RS:0;2eda60943ea1:34197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:42,708 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:51:42,708 INFO [RS:0;2eda60943ea1:34197 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:51:42,708 INFO [RS:0;2eda60943ea1:34197 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T18:51:42,709 INFO [RS:0;2eda60943ea1:34197 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:51:42,709 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:51:42,709 INFO [RS:0;2eda60943ea1:34197 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34197 2024-11-23T18:51:42,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,34197,1732387877820 2024-11-23T18:51:42,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:51:42,829 INFO [RS:0;2eda60943ea1:34197 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:51:42,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,34197,1732387877820] 2024-11-23T18:51:42,860 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,34197,1732387877820 already deleted, retry=false 2024-11-23T18:51:42,860 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,34197,1732387877820 expired; onlineServers=0 2024-11-23T18:51:42,860 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,36073,1732387877629' ***** 2024-11-23T18:51:42,861 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:51:42,861 INFO [M:0;2eda60943ea1:36073 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:51:42,861 INFO [M:0;2eda60943ea1:36073 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:51:42,861 DEBUG [M:0;2eda60943ea1:36073 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:51:42,862 DEBUG [M:0;2eda60943ea1:36073 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:51:42,862 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:51:42,862 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387878629 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387878629,5,FailOnTimeoutGroup] 2024-11-23T18:51:42,862 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387878629 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387878629,5,FailOnTimeoutGroup] 2024-11-23T18:51:42,863 INFO [M:0;2eda60943ea1:36073 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:51:42,863 INFO [M:0;2eda60943ea1:36073 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:51:42,863 DEBUG [M:0;2eda60943ea1:36073 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:51:42,864 INFO [M:0;2eda60943ea1:36073 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:51:42,864 INFO [M:0;2eda60943ea1:36073 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:51:42,864 INFO [M:0;2eda60943ea1:36073 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:51:42,864 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:51:42,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:51:42,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:42,871 DEBUG [M:0;2eda60943ea1:36073 {}] zookeeper.ZKUtil(347): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:51:42,871 WARN [M:0;2eda60943ea1:36073 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:51:42,873 INFO [M:0;2eda60943ea1:36073 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/.lastflushedseqids 2024-11-23T18:51:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741844_1022 (size=139) 2024-11-23T18:51:42,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34391 is added to blk_1073741844_1022 (size=139) 2024-11-23T18:51:42,882 INFO [M:0;2eda60943ea1:36073 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:51:42,882 INFO [M:0;2eda60943ea1:36073 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:51:42,882 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:51:42,882 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:42,883 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:42,883 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:51:42,883 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:42,883 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.08 KB heapSize=29.02 KB 2024-11-23T18:51:42,884 WARN [AsyncFSWAL-0-hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData-prefix:2eda60943ea1,36073,1732387877629 {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.sync(AsyncProtobufLogWriter.java:153) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:159) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doWriterSync(AsyncFSWAL.java:100) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.sync(AbstractFSWAL.java:1629) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1861) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:42,885 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2eda60943ea1%2C36073%2C1732387877629:(num 1732387878360) roll requested 2024-11-23T18:51:42,899 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387902885, exclude list is [], retry=0 2024-11-23T18:51:42,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-93871931-11b0-4641-89ca-81f04a481170,DISK] 2024-11-23T18:51:42,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-ff9df5de-0527-4946-b726-e4069b4967c4,DISK] 2024-11-23T18:51:42,904 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 with entries=54, filesize=26.79 KB; new WAL /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387902885 2024-11-23T18:51:42,904 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38679:38679),(127.0.0.1/127.0.0.1:39817:39817)] 2024-11-23T18:51:42,904 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 is not closed yet, will try archiving it next time 2024-11-23T18:51:42,904 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush0(FanOutOneBlockAsyncDFSOutput.java:469) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.flush(FanOutOneBlockAsyncDFSOutput.java:558) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.lambda$writeWALTrailerAndMagic$3(AsyncProtobufLogWriter.java:249) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALMetadata(AsyncProtobufLogWriter.java:203) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeWALTrailerAndMagic(AsyncProtobufLogWriter.java:240) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.writeWALTrailer(AbstractProtobufLogWriter.java:252) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:162) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:42,905 WARN [Close-WAL-Writer-0 {}] wal.AsyncProtobufLogWriter(165): normal close failed, try recover java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:163) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:42,905 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 2024-11-23T18:51:42,906 WARN [IPC Server handler 3 on default port 38745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741830_1006 2024-11-23T18:51:42,906 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 after 1ms 2024-11-23T18:51:42,922 DEBUG [M:0;2eda60943ea1:36073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4996b358041e435ea8f79a61b4c73ce0 is 82, key is hbase:meta,,1/info:regioninfo/1732387879383/Put/seqid=0 2024-11-23T18:51:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741846_1025 (size=5672) 2024-11-23T18:51:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34391 is added to blk_1073741846_1025 (size=5672) 2024-11-23T18:51:42,928 INFO [M:0;2eda60943ea1:36073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4996b358041e435ea8f79a61b4c73ce0 2024-11-23T18:51:42,949 DEBUG [M:0;2eda60943ea1:36073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b95f6711a1ba433293e6943f811980bd is 773, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387882209/Put/seqid=0 2024-11-23T18:51:42,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:42,950 INFO [RS:0;2eda60943ea1:34197 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:51:42,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34197-0x10169092e970001, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:42,950 INFO [RS:0;2eda60943ea1:34197 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,34197,1732387877820; zookeeper connection closed. 2024-11-23T18:51:42,950 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c714ba9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c714ba9 2024-11-23T18:51:42,950 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T18:51:42,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741847_1026 (size=6113) 2024-11-23T18:51:42,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34391 is added to blk_1073741847_1026 (size=6113) 2024-11-23T18:51:42,956 INFO [M:0;2eda60943ea1:36073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.48 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b95f6711a1ba433293e6943f811980bd 2024-11-23T18:51:42,978 DEBUG [M:0;2eda60943ea1:36073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f687c6b9241b49d095513ff0d02b2142 is 69, key is 2eda60943ea1,34197,1732387877820/rs:state/1732387878687/Put/seqid=0 2024-11-23T18:51:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34391 is added to blk_1073741848_1027 (size=5156) 2024-11-23T18:51:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741848_1027 (size=5156) 2024-11-23T18:51:42,987 INFO [M:0;2eda60943ea1:36073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f687c6b9241b49d095513ff0d02b2142 2024-11-23T18:51:43,007 DEBUG [M:0;2eda60943ea1:36073 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c52bfb2df4274bb083eef774dfdf6865 is 52, key is load_balancer_on/state:d/1732387879564/Put/seqid=0 2024-11-23T18:51:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39117 is added to blk_1073741849_1028 (size=5056) 2024-11-23T18:51:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44603 is added to blk_1073741849_1028 (size=5056) 2024-11-23T18:51:43,014 INFO [M:0;2eda60943ea1:36073 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c52bfb2df4274bb083eef774dfdf6865 2024-11-23T18:51:43,022 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4996b358041e435ea8f79a61b4c73ce0 as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4996b358041e435ea8f79a61b4c73ce0 2024-11-23T18:51:43,029 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4996b358041e435ea8f79a61b4c73ce0, entries=8, sequenceid=56, filesize=5.5 K 2024-11-23T18:51:43,031 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b95f6711a1ba433293e6943f811980bd as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b95f6711a1ba433293e6943f811980bd 2024-11-23T18:51:43,037 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b95f6711a1ba433293e6943f811980bd, entries=6, sequenceid=56, filesize=6.0 K 2024-11-23T18:51:43,038 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f687c6b9241b49d095513ff0d02b2142 as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f687c6b9241b49d095513ff0d02b2142 2024-11-23T18:51:43,044 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f687c6b9241b49d095513ff0d02b2142, entries=1, sequenceid=56, filesize=5.0 K 2024-11-23T18:51:43,045 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c52bfb2df4274bb083eef774dfdf6865 as hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c52bfb2df4274bb083eef774dfdf6865 2024-11-23T18:51:43,051 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c52bfb2df4274bb083eef774dfdf6865, entries=1, sequenceid=56, filesize=4.9 K 2024-11-23T18:51:43,052 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.08 KB/23632, heapSize ~28.96 KB/29656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=56, compaction requested=false 2024-11-23T18:51:43,054 INFO [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:43,054 DEBUG [M:0;2eda60943ea1:36073 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387902882Disabling compacts and flushes for region at 1732387902882Disabling writes for close at 1732387902883 (+1 ms)Obtaining lock to block concurrent updates at 1732387902883Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732387902883Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23632, getHeapSize=29656, getOffHeapSize=0, getCellsCount=67 at 1732387902884 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732387902905 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732387902905Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732387902921 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732387902921Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732387902935 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732387902949 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732387902949Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732387902962 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732387902978 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732387902978Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732387902993 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732387903007 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732387903007Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aff7b5b: reopening flushed file at 1732387903021 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@82a8527: reopening flushed file at 1732387903029 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c740a6: reopening flushed file at 1732387903037 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c2b82b1: reopening flushed file at 1732387903044 (+7 ms)Finished flush of dataSize ~23.08 KB/23632, heapSize ~28.96 KB/29656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=56, compaction requested=false at 1732387903052 (+8 ms)Writing region close event to WAL at 1732387903053 (+1 ms)Closed at 1732387903054 (+1 ms) 2024-11-23T18:51:43,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39117 is added to blk_1073741845_1023 (size=757) 2024-11-23T18:51:43,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741845_1023 (size=757) 2024-11-23T18:51:43,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:43,255 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.InterruptedIOException: Operation cancelled at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.checkIfCancelled(RecoverLeaseFSUtils.java:269) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:190) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:43,256 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 2024-11-23T18:51:43,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=0 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 after 0ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:43,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for /user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 13 more 2024-11-23T18:51:43,451 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.InterruptedIOException: Operation cancelled at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.checkIfCancelled(RecoverLeaseFSUtils.java:269) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:190) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.recoverAndClose(FanOutOneBlockAsyncDFSOutput.java:605) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.close(AsyncProtobufLogWriter.java:166) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2041) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:43,452 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta 2024-11-23T18:51:43,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=0 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta after 0ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:44,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741833_1014 (size=1599) 2024-11-23T18:51:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44293 is added to blk_1073741830_1024 (size=27165) 2024-11-23T18:51:46,908 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 after 4003ms 2024-11-23T18:51:46,910 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/WALs/2eda60943ea1,36073,1732387877629/2eda60943ea1%2C36073%2C1732387877629.1732387878360 to hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/oldWALs/2eda60943ea1%2C36073%2C1732387877629.1732387878360 2024-11-23T18:51:46,918 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/MasterData/oldWALs/2eda60943ea1%2C36073%2C1732387877629.1732387878360 to hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/oldWALs/2eda60943ea1%2C36073%2C1732387877629.1732387878360$masterlocalwal$ 2024-11-23T18:51:46,918 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:51:46,918 INFO [M:0;2eda60943ea1:36073 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:51:46,918 INFO [M:0;2eda60943ea1:36073 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36073 2024-11-23T18:51:46,918 INFO [M:0;2eda60943ea1:36073 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:51:47,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:47,076 INFO [M:0;2eda60943ea1:36073 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:51:47,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36073-0x10169092e970000, quorum=127.0.0.1:61095, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:51:47,115 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7aba7c39{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:47,115 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@387fbdda{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,115 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,116 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dbae964{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,116 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f102659{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,119 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:47,119 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:47,119 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid 01d6c3f6-3515-48a5-9cd0-b8ec8155928e) service to localhost/127.0.0.1:38745 2024-11-23T18:51:47,119 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:47,120 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data1/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,121 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data2/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,121 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:47,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@245d91f8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:47,125 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3bc43b80{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,125 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,125 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28ab81a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,125 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34d98298{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,126 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:47,126 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:47,126 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid 3a531745-f7a0-4e15-a187-c12e49aa3c8c) service to localhost/127.0.0.1:38745 2024-11-23T18:51:47,126 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:47,127 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data9/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,127 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data10/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,127 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:47,129 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ea0ce88{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:47,129 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27724107{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,129 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,129 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71aa8073{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,129 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2037e379{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,130 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:47,130 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:47,130 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:47,130 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid b600b879-1003-484c-8323-7488f2a57195) service to localhost/127.0.0.1:38745 2024-11-23T18:51:47,131 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data7/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,131 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data8/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,131 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:47,133 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2874572a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:47,133 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8b61256{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,133 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,133 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a1625d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,133 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ee815b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,134 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:47,134 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:47,134 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid d5a133dd-fbd4-46d8-a038-7fd4418d0a06) service to localhost/127.0.0.1:38745 2024-11-23T18:51:47,134 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:47,135 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data5/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,135 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data6/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,136 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:47,142 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f4810cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:47,142 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@18152f49{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,142 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,143 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@679249d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,143 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f4a76c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,144 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:51:47,144 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:51:47,144 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:51:47,144 WARN [BP-1056060911-172.17.0.2-1732387873601 heartbeating to localhost/127.0.0.1:38745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1056060911-172.17.0.2-1732387873601 (Datanode Uuid 60f2802f-1415-4d27-ab88-d73cab2efbbb) service to localhost/127.0.0.1:38745 2024-11-23T18:51:47,144 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@14f1938f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1056060911-172.17.0.2-1732387873601:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45909,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:38649 , LocalHost:localPort 2eda60943ea1/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-23T18:51:47,145 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@14f1938f {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1056060911-172.17.0.2-1732387873601:blk_1073741834_1010; getBlockSize()=0; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:44293,null,null], DatanodeInfoWithStorage[127.0.0.1:45909,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1056060911-172.17.0.2-1732387873601 2024-11-23T18:51:47,145 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data3/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,145 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/cluster_9cc3adde-fe81-3d50-146f-4ab1ea033b86/data/data4/current/BP-1056060911-172.17.0.2-1732387873601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:51:47,145 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:51:47,151 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a9f0012{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:51:47,151 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@724b4e38{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:51:47,152 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:51:47,152 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@143ef478{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:51:47,152 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b1b5aec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir/,STOPPED} 2024-11-23T18:51:47,160 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:51:47,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T18:51:47,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRollOnDatanodeDeath Thread=102 (was 66) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38745 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:166) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$908/0x00007f60d8bf7390.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:38745 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-13-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:166) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$908/0x00007f60d8bf7390.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:38745 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-13-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38745 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38745 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (961219246) connection to localhost/127.0.0.1:38745 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-10-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 393) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=106 (was 144), ProcessCount=11 (was 11), AvailableMemoryMB=8894 (was 9044) 2024-11-23T18:51:47,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testCompactionRecordDoesntBlockRolling Thread=102, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=106, ProcessCount=11, AvailableMemoryMB=8893 2024-11-23T18:51:47,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:51:47,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.log.dir so I do NOT create it in target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a96815a-1477-7a2c-205a-42bf62767a0d/hadoop.tmp.dir so I do NOT create it in target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34, deleteOnExit=true 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/test.cache.data in system properties and HBase conf 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:51:47,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:51:47,204 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:51:47,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:51:47,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:51:47,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=1 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 after 4001ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:47,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=1 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta after 4001ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:51:47,504 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T18:51:47,506 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,519 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,520 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,520 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,521 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,521 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,521 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,525 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:47,543 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:47,549 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:47,551 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:47,551 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:47,551 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:51:47,551 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:47,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55b966d7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:47,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4475c417{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:47,644 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18e25308{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/java.io.tmpdir/jetty-localhost-36611-hadoop-hdfs-3_4_1-tests_jar-_-any-7396971804398634588/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:51:47,645 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5675ef4f{HTTP/1.1, (http/1.1)}{localhost:36611} 2024-11-23T18:51:47,645 INFO [Time-limited test {}] server.Server(415): Started @101795ms 2024-11-23T18:51:47,937 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:47,942 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:47,943 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:47,943 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:47,943 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:51:47,944 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@440a68df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:47,944 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@457d2735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:48,038 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@628df22e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/java.io.tmpdir/jetty-localhost-38227-hadoop-hdfs-3_4_1-tests_jar-_-any-15877658789310023735/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:48,038 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1db674f6{HTTP/1.1, (http/1.1)}{localhost:38227} 2024-11-23T18:51:48,038 INFO [Time-limited test {}] server.Server(415): Started @102188ms 2024-11-23T18:51:48,040 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:48,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:51:48,087 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:51:48,088 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:51:48,088 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:51:48,088 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:51:48,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f540c00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:51:48,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d453491{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:51:48,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fcd28da{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/java.io.tmpdir/jetty-localhost-42579-hadoop-hdfs-3_4_1-tests_jar-_-any-7123768963762331573/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:51:48,180 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5fac4361{HTTP/1.1, (http/1.1)}{localhost:42579} 2024-11-23T18:51:48,181 INFO [Time-limited test {}] server.Server(415): Started @102330ms 2024-11-23T18:51:48,182 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:51:48,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-23T18:51:48,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 12 more 2024-11-23T18:51:49,154 WARN [Thread-802 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data1/current/BP-1133735507-172.17.0.2-1732387907229/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:49,154 WARN [Thread-803 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data2/current/BP-1133735507-172.17.0.2-1732387907229/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:49,175 WARN [Thread-766 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:49,177 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bed325938c30277 with lease ID 0x74dd372f9a292f72: Processing first storage report for DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b from datanode DatanodeRegistration(127.0.0.1:39103, datanodeUuid=2cbc6015-887b-4c17-870d-44967159fdda, infoPort=35223, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229) 2024-11-23T18:51:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bed325938c30277 with lease ID 0x74dd372f9a292f72: from storage DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b node DatanodeRegistration(127.0.0.1:39103, datanodeUuid=2cbc6015-887b-4c17-870d-44967159fdda, infoPort=35223, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bed325938c30277 with lease ID 0x74dd372f9a292f72: Processing first storage report for DS-923461d6-da74-4341-991a-42fd6b803752 from datanode DatanodeRegistration(127.0.0.1:39103, datanodeUuid=2cbc6015-887b-4c17-870d-44967159fdda, infoPort=35223, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229) 2024-11-23T18:51:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bed325938c30277 with lease ID 0x74dd372f9a292f72: from storage DS-923461d6-da74-4341-991a-42fd6b803752 node DatanodeRegistration(127.0.0.1:39103, datanodeUuid=2cbc6015-887b-4c17-870d-44967159fdda, infoPort=35223, infoSecurePort=0, ipcPort=37593, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:49,259 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:49,303 WARN [Thread-813 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data3/current/BP-1133735507-172.17.0.2-1732387907229/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:49,303 WARN [Thread-814 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data4/current/BP-1133735507-172.17.0.2-1732387907229/current, will proceed with Du for space computation calculation, 2024-11-23T18:51:49,325 WARN [Thread-789 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:51:49,327 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb52aa8ef5e17e4f4 with lease ID 0x74dd372f9a292f73: Processing first storage report for DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c from datanode DatanodeRegistration(127.0.0.1:44389, datanodeUuid=72fee641-54b7-4b6f-a5ad-ae5bc4ea6750, infoPort=34803, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229) 2024-11-23T18:51:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb52aa8ef5e17e4f4 with lease ID 0x74dd372f9a292f73: from storage DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c node DatanodeRegistration(127.0.0.1:44389, datanodeUuid=72fee641-54b7-4b6f-a5ad-ae5bc4ea6750, infoPort=34803, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb52aa8ef5e17e4f4 with lease ID 0x74dd372f9a292f73: Processing first storage report for DS-7cc6bdcc-e0f0-4a23-b4ba-94143bdff9ec from datanode DatanodeRegistration(127.0.0.1:44389, datanodeUuid=72fee641-54b7-4b6f-a5ad-ae5bc4ea6750, infoPort=34803, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229) 2024-11-23T18:51:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb52aa8ef5e17e4f4 with lease ID 0x74dd372f9a292f73: from storage DS-7cc6bdcc-e0f0-4a23-b4ba-94143bdff9ec node DatanodeRegistration(127.0.0.1:44389, datanodeUuid=72fee641-54b7-4b6f-a5ad-ae5bc4ea6750, infoPort=34803, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=1078919092;c=1732387907229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:51:49,422 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131 2024-11-23T18:51:49,428 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/zookeeper_0, clientPort=59128, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:51:49,429 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59128 2024-11-23T18:51:49,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,430 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:51:49,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:51:49,440 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3 with version=8 2024-11-23T18:51:49,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase-staging 2024-11-23T18:51:49,442 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:51:49,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,442 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:51:49,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:51:49,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:51:49,443 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:51:49,443 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37245 2024-11-23T18:51:49,445 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37245 connecting to ZooKeeper ensemble=127.0.0.1:59128 2024-11-23T18:51:49,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:49,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:372450x0, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:51:49,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37245-0x1016909aade0000 connected 2024-11-23T18:51:49,586 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:49,590 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3, hbase.cluster.distributed=false 2024-11-23T18:51:49,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:51:49,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37245 2024-11-23T18:51:49,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37245 2024-11-23T18:51:49,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37245 2024-11-23T18:51:49,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37245 2024-11-23T18:51:49,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37245 2024-11-23T18:51:49,610 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:51:49,610 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:51:49,611 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40689 2024-11-23T18:51:49,612 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40689 connecting to ZooKeeper ensemble=127.0.0.1:59128 2024-11-23T18:51:49,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406890x0, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:51:49,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40689-0x1016909aade0001 connected 2024-11-23T18:51:49,629 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:51:49,629 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:51:49,630 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:51:49,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:51:49,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:51:49,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40689 2024-11-23T18:51:49,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40689 2024-11-23T18:51:49,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40689 2024-11-23T18:51:49,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40689 2024-11-23T18:51:49,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40689 2024-11-23T18:51:49,647 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:37245 2024-11-23T18:51:49,648 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,37245,1732387909442 2024-11-23T18:51:49,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:49,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:49,660 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,37245,1732387909442 2024-11-23T18:51:49,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:51:49,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:49,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:49,670 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:51:49,671 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,37245,1732387909442 from backup master directory 2024-11-23T18:51:49,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,37245,1732387909442 2024-11-23T18:51:49,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:49,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:51:49,680 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:51:49,681 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,37245,1732387909442 2024-11-23T18:51:49,689 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/hbase.id] with ID: c9e12196-ca8e-4bc2-afaf-a6c2f855218a 2024-11-23T18:51:49,689 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/.tmp/hbase.id 2024-11-23T18:51:49,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:51:49,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:51:49,696 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/.tmp/hbase.id]:[hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/hbase.id] 2024-11-23T18:51:49,708 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:49,708 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:51:49,710 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T18:51:49,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:49,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:51:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:51:50,135 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:51:50,137 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:51:50,138 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:50,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:51:50,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:51:50,149 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store 2024-11-23T18:51:50,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:51:50,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:51:50,157 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:50,157 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:51:50,157 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387910157Disabling compacts and flushes for region at 1732387910157Disabling writes for close at 1732387910157Writing region close event to WAL at 1732387910157Closed at 1732387910157 2024-11-23T18:51:50,158 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/.initializing 2024-11-23T18:51:50,158 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/WALs/2eda60943ea1,37245,1732387909442 2024-11-23T18:51:50,159 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:51:50,161 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C37245%2C1732387909442, suffix=, logDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/WALs/2eda60943ea1,37245,1732387909442, archiveDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/oldWALs, maxLogs=10 2024-11-23T18:51:50,173 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/WALs/2eda60943ea1,37245,1732387909442/2eda60943ea1%2C37245%2C1732387909442.1732387910161, exclude list is [], retry=0 2024-11-23T18:51:50,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:51:50,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:51:50,178 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/WALs/2eda60943ea1,37245,1732387909442/2eda60943ea1%2C37245%2C1732387909442.1732387910161 2024-11-23T18:51:50,179 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34803:34803),(127.0.0.1/127.0.0.1:35223:35223)] 2024-11-23T18:51:50,179 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:50,179 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:50,179 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,179 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:51:50,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:50,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:51:50,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:50,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:51:50,186 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:50,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:51:50,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:50,189 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,190 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,191 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,192 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,192 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,193 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:51:50,194 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:51:50,196 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:50,196 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=837689, jitterRate=0.06517699360847473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:51:50,197 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387910179Initializing all the Stores at 1732387910180 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387910180Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387910181 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387910181Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387910181Cleaning up temporary data from old regions at 1732387910192 (+11 ms)Region opened successfully at 1732387910197 (+5 ms) 2024-11-23T18:51:50,197 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:51:50,201 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234cf6a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:51:50,202 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:51:50,202 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:51:50,202 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:51:50,202 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:51:50,203 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T18:51:50,203 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T18:51:50,203 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:51:50,205 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:51:50,206 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:51:50,249 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:51:50,249 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:51:50,251 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:51:50,259 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:51:50,260 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:51:50,261 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:50,262 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:51:50,270 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:51:50,272 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:51:50,280 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:51:50,282 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:51:50,291 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:51:50,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:50,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:51:50,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,302 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,37245,1732387909442, sessionid=0x1016909aade0000, setting cluster-up flag (Was=false) 2024-11-23T18:51:50,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,354 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:51:50,356 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37245,1732387909442 2024-11-23T18:51:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:50,407 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:51:50,409 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37245,1732387909442 2024-11-23T18:51:50,411 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:51:50,413 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:50,413 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:51:50,413 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:51:50,414 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,37245,1732387909442 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,416 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:51:50,417 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,418 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387940418 2024-11-23T18:51:50,418 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:51:50,418 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:51:50,418 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:51:50,418 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:51:50,419 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:51:50,419 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:51:50,419 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,419 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:50,419 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:51:50,420 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:51:50,420 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:51:50,420 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:51:50,420 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:51:50,420 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:51:50,421 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387910421,5,FailOnTimeoutGroup] 2024-11-23T18:51:50,421 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387910421,5,FailOnTimeoutGroup] 2024-11-23T18:51:50,421 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,421 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:51:50,422 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,422 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,422 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,422 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:51:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:51:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:51:50,433 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:51:50,434 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3 2024-11-23T18:51:50,437 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(746): ClusterId : c9e12196-ca8e-4bc2-afaf-a6c2f855218a 2024-11-23T18:51:50,437 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:51:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:51:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:51:50,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:50,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:51:50,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:51:50,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:50,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:51:50,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:51:50,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:50,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:51:50,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:51:50,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:50,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:51:50,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:51:50,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:50,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:50,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:51:50,450 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:51:50,450 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:51:50,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740 2024-11-23T18:51:50,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740 2024-11-23T18:51:50,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:51:50,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:51:50,452 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:51:50,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:51:50,455 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:50,456 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804161, jitterRate=0.022544533014297485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:51:50,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387910441Initializing all the Stores at 1732387910441Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387910441Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387910442 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387910442Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387910442Cleaning up temporary data from old regions at 1732387910451 (+9 ms)Region opened successfully at 1732387910456 (+5 ms) 2024-11-23T18:51:50,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:51:50,456 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:51:50,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:51:50,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:51:50,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:51:50,457 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:51:50,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:50,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387910456Disabling compacts and flushes for region at 1732387910456Disabling writes for close at 1732387910456Writing region close event to WAL at 1732387910457 (+1 ms)Closed at 1732387910457 2024-11-23T18:51:50,459 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:50,459 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:51:50,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:51:50,460 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:51:50,460 DEBUG [RS:0;2eda60943ea1:40689 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6aaab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:51:50,460 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:51:50,462 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:51:50,472 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:40689 2024-11-23T18:51:50,473 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:51:50,473 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:51:50,473 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:51:50,473 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37245,1732387909442 with port=40689, startcode=1732387909610 2024-11-23T18:51:50,474 DEBUG [RS:0;2eda60943ea1:40689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:51:50,475 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55353, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:51:50,476 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,476 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37245 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,477 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3 2024-11-23T18:51:50,478 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43611 2024-11-23T18:51:50,478 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:51:50,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:51:50,491 DEBUG [RS:0;2eda60943ea1:40689 {}] zookeeper.ZKUtil(111): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,491 WARN [RS:0;2eda60943ea1:40689 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:51:50,491 INFO [RS:0;2eda60943ea1:40689 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:50,491 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,491 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,40689,1732387909610] 2024-11-23T18:51:50,495 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:51:50,498 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:51:50,498 INFO [RS:0;2eda60943ea1:40689 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:51:50,498 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,499 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:51:50,500 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:51:50,500 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,500 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,500 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,500 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,500 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,500 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:51:50,501 DEBUG [RS:0;2eda60943ea1:40689 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,502 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,40689,1732387909610-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:51:50,518 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:51:50,518 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,40689,1732387909610-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,518 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,518 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.Replication(171): 2eda60943ea1,40689,1732387909610 started 2024-11-23T18:51:50,531 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:50,531 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,40689,1732387909610, RpcServer on 2eda60943ea1/172.17.0.2:40689, sessionid=0x1016909aade0001 2024-11-23T18:51:50,531 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:51:50,531 DEBUG [RS:0;2eda60943ea1:40689 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,531 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,40689,1732387909610' 2024-11-23T18:51:50,531 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,40689,1732387909610' 2024-11-23T18:51:50,532 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:51:50,533 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:51:50,533 DEBUG [RS:0;2eda60943ea1:40689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:51:50,533 INFO [RS:0;2eda60943ea1:40689 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:51:50,533 INFO [RS:0;2eda60943ea1:40689 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:51:50,612 WARN [2eda60943ea1:37245 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:51:50,634 INFO [RS:0;2eda60943ea1:40689 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:51:50,637 INFO [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C40689%2C1732387909610, suffix=, logDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610, archiveDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs, maxLogs=32 2024-11-23T18:51:50,653 DEBUG [RS:0;2eda60943ea1:40689 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638, exclude list is [], retry=0 2024-11-23T18:51:50,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:51:50,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:51:50,657 INFO [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638 2024-11-23T18:51:50,658 DEBUG [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35223:35223),(127.0.0.1/127.0.0.1:34803:34803)] 2024-11-23T18:51:50,863 DEBUG [2eda60943ea1:37245 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T18:51:50,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:50,867 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,40689,1732387909610, state=OPENING 2024-11-23T18:51:51,017 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:51:51,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:51,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:51:51,111 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:51:51,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:51,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:51,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,40689,1732387909610}] 2024-11-23T18:51:51,262 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:51,268 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:51:51,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:51:51,279 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:51:51,279 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:51:51,279 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:51:51,281 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C40689%2C1732387909610.meta, suffix=.meta, logDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610, archiveDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs, maxLogs=32 2024-11-23T18:51:51,293 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.meta.1732387911281.meta, exclude list is [], retry=0 2024-11-23T18:51:51,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:51:51,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:51:51,298 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.meta.1732387911281.meta 2024-11-23T18:51:51,298 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35223:35223),(127.0.0.1/127.0.0.1:34803:34803)] 2024-11-23T18:51:51,298 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:51:51,299 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:51:51,299 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:51:51,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:51:51,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:51:51,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:51,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:51:51,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:51:51,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:51,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:51:51,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:51:51,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:51,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:51:51,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:51:51,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:51:51,306 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:51:51,307 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740 2024-11-23T18:51:51,308 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740 2024-11-23T18:51:51,309 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:51:51,309 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:51:51,310 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:51:51,311 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:51:51,312 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=803764, jitterRate=0.022039875388145447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:51:51,312 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:51:51,313 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387911299Writing region info on filesystem at 1732387911299Initializing all the Stores at 1732387911300 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387911300Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387911300Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387911300Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387911300Cleaning up temporary data from old regions at 1732387911309 (+9 ms)Running coprocessor post-open hooks at 1732387911312 (+3 ms)Region opened successfully at 1732387911313 (+1 ms) 2024-11-23T18:51:51,314 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387911267 2024-11-23T18:51:51,316 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:51:51,316 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:51:51,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:51,317 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,40689,1732387909610, state=OPEN 2024-11-23T18:51:51,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:51:51,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:51:51,373 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:51,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:51,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:51:51,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:51:51,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,40689,1732387909610 in 262 msec 2024-11-23T18:51:51,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:51:51,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 920 msec 2024-11-23T18:51:51,385 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:51:51,385 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:51:51,387 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:51:51,387 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,40689,1732387909610, seqNum=-1] 2024-11-23T18:51:51,387 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:51:51,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44757, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:51:51,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 982 msec 2024-11-23T18:51:51,396 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387911395, completionTime=-1 2024-11-23T18:51:51,396 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T18:51:51,396 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:51:51,398 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T18:51:51,398 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732387971398 2024-11-23T18:51:51,398 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732388031398 2024-11-23T18:51:51,398 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:37245, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,399 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,402 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.724sec 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:51:51,405 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:51:51,409 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:51:51,410 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:51:51,410 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37245,1732387909442-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:51:51,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ac8bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:51,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,37245,-1 for getting cluster id 2024-11-23T18:51:51,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:51:51,442 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c9e12196-ca8e-4bc2-afaf-a6c2f855218a' 2024-11-23T18:51:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:51:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c9e12196-ca8e-4bc2-afaf-a6c2f855218a" 2024-11-23T18:51:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a49cf0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,37245,-1] 2024-11-23T18:51:51,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:51:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:51:51,447 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:51:51,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e81cfe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:51:51,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:51:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,40689,1732387909610, seqNum=-1] 2024-11-23T18:51:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:51:51,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:51:51,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,37245,1732387909442 2024-11-23T18:51:51,455 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:51:51,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:51,458 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-23T18:51:51,458 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:51:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,37245,1732387909442 2024-11-23T18:51:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@614dca71 2024-11-23T18:51:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:51:51,462 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:51:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-23T18:51:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-23T18:51:51,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:51:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:51:51,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:51:51,466 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-23T18:51:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:51:51,468 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:51:51,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741835_1011 (size=405) 2024-11-23T18:51:51,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741835_1011 (size=405) 2024-11-23T18:51:51,477 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 11883101a4f15c51f8988731ba67e2e0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3 2024-11-23T18:51:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741836_1012 (size=88) 2024-11-23T18:51:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741836_1012 (size=88) 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 11883101a4f15c51f8988731ba67e2e0, disabling compactions & flushes 2024-11-23T18:51:51,485 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. after waiting 0 ms 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,485 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,485 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 11883101a4f15c51f8988731ba67e2e0: Waiting for close lock at 1732387911485Disabling compacts and flushes for region at 1732387911485Disabling writes for close at 1732387911485Writing region close event to WAL at 1732387911485Closed at 1732387911485 2024-11-23T18:51:51,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:51:51,487 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732387911487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387911487"}]},"ts":"1732387911487"} 2024-11-23T18:51:51,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:51:51,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:51:51,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387911491"}]},"ts":"1732387911491"} 2024-11-23T18:51:51,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-23T18:51:51,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=11883101a4f15c51f8988731ba67e2e0, ASSIGN}] 2024-11-23T18:51:51,495 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=11883101a4f15c51f8988731ba67e2e0, ASSIGN 2024-11-23T18:51:51,496 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=11883101a4f15c51f8988731ba67e2e0, ASSIGN; state=OFFLINE, location=2eda60943ea1,40689,1732387909610; forceNewPlan=false, retain=false 2024-11-23T18:51:51,647 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=11883101a4f15c51f8988731ba67e2e0, regionState=OPENING, regionLocation=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:51,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=11883101a4f15c51f8988731ba67e2e0, ASSIGN because future has completed 2024-11-23T18:51:51,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11883101a4f15c51f8988731ba67e2e0, server=2eda60943ea1,40689,1732387909610}] 2024-11-23T18:51:51,819 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,819 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 11883101a4f15c51f8988731ba67e2e0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:51:51,820 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,820 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:51:51,820 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,821 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,823 INFO [StoreOpener-11883101a4f15c51f8988731ba67e2e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,825 INFO [StoreOpener-11883101a4f15c51f8988731ba67e2e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11883101a4f15c51f8988731ba67e2e0 columnFamilyName info 2024-11-23T18:51:51,825 DEBUG [StoreOpener-11883101a4f15c51f8988731ba67e2e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:51:51,826 INFO [StoreOpener-11883101a4f15c51f8988731ba67e2e0-1 {}] regionserver.HStore(327): Store=11883101a4f15c51f8988731ba67e2e0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:51:51,826 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,827 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,827 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,828 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,828 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,831 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,834 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:51:51,835 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 11883101a4f15c51f8988731ba67e2e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=808346, jitterRate=0.027866095304489136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:51:51,835 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:51:51,836 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 11883101a4f15c51f8988731ba67e2e0: Running coprocessor pre-open hook at 1732387911821Writing region info on filesystem at 1732387911821Initializing all the Stores at 1732387911822 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387911822Cleaning up temporary data from old regions at 1732387911828 (+6 ms)Running coprocessor post-open hooks at 1732387911835 (+7 ms)Region opened successfully at 1732387911836 (+1 ms) 2024-11-23T18:51:51,837 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0., pid=6, masterSystemTime=1732387911811 2024-11-23T18:51:51,840 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,840 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:51:51,842 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=11883101a4f15c51f8988731ba67e2e0, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,40689,1732387909610 2024-11-23T18:51:51,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11883101a4f15c51f8988731ba67e2e0, server=2eda60943ea1,40689,1732387909610 because future has completed 2024-11-23T18:51:51,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:51:51,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 11883101a4f15c51f8988731ba67e2e0, server=2eda60943ea1,40689,1732387909610 in 193 msec 2024-11-23T18:51:51,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:51:51,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=11883101a4f15c51f8988731ba67e2e0, ASSIGN in 358 msec 2024-11-23T18:51:51,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:51:51,857 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387911856"}]},"ts":"1732387911856"} 2024-11-23T18:51:51,859 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-23T18:51:51,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:51:51,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 398 msec 2024-11-23T18:51:52,263 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:52,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:52,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:51:52,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T18:51:52,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:51:52,465 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T18:51:52,466 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:51:52,466 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-23T18:51:53,265 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:53,460 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:54,266 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:54,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:55,268 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:55,463 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:56,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:56,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:56,805 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:51:56,824 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,826 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,829 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,831 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:51:56,837 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:51:56,838 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-23T18:51:57,271 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:57,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:58,272 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:58,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:59,274 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:51:59,469 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:00,276 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:00,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:01,277 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:01,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:52:01,554 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-23T18:52:01,554 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-23T18:52:01,562 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:01,562 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0., hostname=2eda60943ea1,40689,1732387909610, seqNum=2] 2024-11-23T18:52:01,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:01,579 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-23T18:52:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T18:52:01,581 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T18:52:01,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T18:52:01,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T18:52:01,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:01,752 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 11883101a4f15c51f8988731ba67e2e0 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-23T18:52:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/8a7e9a05389349b5a2c79cef6f20825b is 1055, key is row0001/info:/1732387921568/Put/seqid=0 2024-11-23T18:52:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741837_1013 (size=6008) 2024-11-23T18:52:01,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741837_1013 (size=6008) 2024-11-23T18:52:01,773 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/8a7e9a05389349b5a2c79cef6f20825b 2024-11-23T18:52:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/8a7e9a05389349b5a2c79cef6f20825b as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b 2024-11-23T18:52:01,788 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b, entries=1, sequenceid=5, filesize=5.9 K 2024-11-23T18:52:01,789 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 37ms, sequenceid=5, compaction requested=false 2024-11-23T18:52:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 11883101a4f15c51f8988731ba67e2e0: 2024-11-23T18:52:01,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:01,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T18:52:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T18:52:01,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T18:52:01,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec 2024-11-23T18:52:01,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 223 msec 2024-11-23T18:52:02,279 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:02,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:03,281 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:03,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:04,282 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:04,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:05,284 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:05,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:06,285 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:06,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:07,286 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:07,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:08,287 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:08,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:09,289 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:09,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:10,290 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:10,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:11,291 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:11,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T18:52:11,594 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-23T18:52:11,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T18:52:11,604 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-23T18:52:11,605 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T18:52:11,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T18:52:11,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T18:52:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:11,758 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 11883101a4f15c51f8988731ba67e2e0 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-23T18:52:11,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/a22f44860d104e33826e8ca7079c3239 is 1055, key is row0002/info:/1732387931598/Put/seqid=0 2024-11-23T18:52:11,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741838_1014 (size=6008) 2024-11-23T18:52:11,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741838_1014 (size=6008) 2024-11-23T18:52:11,768 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/a22f44860d104e33826e8ca7079c3239 2024-11-23T18:52:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/a22f44860d104e33826e8ca7079c3239 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239 2024-11-23T18:52:11,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239, entries=1, sequenceid=9, filesize=5.9 K 2024-11-23T18:52:11,783 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 25ms, sequenceid=9, compaction requested=false 2024-11-23T18:52:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 11883101a4f15c51f8988731ba67e2e0: 2024-11-23T18:52:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-23T18:52:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-23T18:52:11,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-23T18:52:11,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec 2024-11-23T18:52:11,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 187 msec 2024-11-23T18:52:12,292 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:12,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:13,294 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:13,485 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:14,295 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:14,487 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:15,296 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:15,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:16,298 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:16,490 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:17,300 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:17,491 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:18,301 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:18,492 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:19,303 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:19,422 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:52:19,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:20,304 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:20,495 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:21,306 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:21,496 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T18:52:21,715 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-23T18:52:21,737 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387941720, exclude list is [], retry=0 2024-11-23T18:52:21,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:52:21,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:52:21,741 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638 with entries=8, filesize=5.33 KB; new WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387941720 2024-11-23T18:52:21,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34803:34803),(127.0.0.1/127.0.0.1:35223:35223)] 2024-11-23T18:52:21,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638 is not closed yet, will try archiving it next time 2024-11-23T18:52:21,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741833_1009 (size=5471) 2024-11-23T18:52:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:21,744 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638 not finished, retry = 0 2024-11-23T18:52:21,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741833_1009 (size=5471) 2024-11-23T18:52:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T18:52:21,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-23T18:52:21,746 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T18:52:21,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T18:52:21,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T18:52:21,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:21,903 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 11883101a4f15c51f8988731ba67e2e0 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-23T18:52:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/654b3fbff88c4371bde9253c4e562780 is 1055, key is row0003/info:/1732387941717/Put/seqid=0 2024-11-23T18:52:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741840_1016 (size=6008) 2024-11-23T18:52:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741840_1016 (size=6008) 2024-11-23T18:52:21,919 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/654b3fbff88c4371bde9253c4e562780 2024-11-23T18:52:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/654b3fbff88c4371bde9253c4e562780 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780 2024-11-23T18:52:21,935 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780, entries=1, sequenceid=13, filesize=5.9 K 2024-11-23T18:52:21,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 34ms, sequenceid=13, compaction requested=true 2024-11-23T18:52:21,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 11883101a4f15c51f8988731ba67e2e0: 2024-11-23T18:52:21,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:21,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-23T18:52:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-23T18:52:21,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T18:52:21,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 192 msec 2024-11-23T18:52:21,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 199 msec 2024-11-23T18:52:22,307 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:22,498 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:23,309 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:23,499 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:24,310 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:24,501 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:25,311 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:25,502 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:26,313 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:26,503 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:27,314 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:27,505 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:28,315 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:28,506 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:29,317 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:29,507 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:30,319 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:30,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:31,320 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:31,410 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T18:52:31,411 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T18:52:31,510 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T18:52:31,775 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-23T18:52:31,783 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:52:31,786 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18024 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:52:31,787 DEBUG [Time-limited test {}] regionserver.HStore(1541): 11883101a4f15c51f8988731ba67e2e0/info is initiating minor compaction (all files) 2024-11-23T18:52:31,787 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:52:31,787 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:31,788 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 11883101a4f15c51f8988731ba67e2e0/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:31,788 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780] into tmpdir=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp, totalSize=17.6 K 2024-11-23T18:52:31,789 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8a7e9a05389349b5a2c79cef6f20825b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732387921568 2024-11-23T18:52:31,789 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a22f44860d104e33826e8ca7079c3239, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732387931598 2024-11-23T18:52:31,790 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 654b3fbff88c4371bde9253c4e562780, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732387941717 2024-11-23T18:52:31,812 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 11883101a4f15c51f8988731ba67e2e0#info#compaction#21 average throughput is 3.01 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:52:31,813 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/3a228e4ad2a84794b5bfa0b4745ada0b is 1055, key is row0001/info:/1732387921568/Put/seqid=0 2024-11-23T18:52:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741841_1017 (size=8221) 2024-11-23T18:52:31,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741841_1017 (size=8221) 2024-11-23T18:52:31,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/3a228e4ad2a84794b5bfa0b4745ada0b as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/3a228e4ad2a84794b5bfa0b4745ada0b 2024-11-23T18:52:31,838 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11883101a4f15c51f8988731ba67e2e0/info of 11883101a4f15c51f8988731ba67e2e0 into 3a228e4ad2a84794b5bfa0b4745ada0b(size=8.0 K), total size for store is 8.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:52:31,838 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 11883101a4f15c51f8988731ba67e2e0: 2024-11-23T18:52:31,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841, exclude list is [], retry=0 2024-11-23T18:52:31,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:52:31,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:52:31,858 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387941720 with entries=4, filesize=2.43 KB; new WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841 2024-11-23T18:52:31,858 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34803:34803),(127.0.0.1/127.0.0.1:35223:35223)] 2024-11-23T18:52:31,859 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387941720 is not closed yet, will try archiving it next time 2024-11-23T18:52:31,859 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387910638 to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs/2eda60943ea1%2C40689%2C1732387909610.1732387910638 2024-11-23T18:52:31,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:31,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741839_1015 (size=2495) 2024-11-23T18:52:31,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741839_1015 (size=2495) 2024-11-23T18:52:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:31,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-23T18:52:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T18:52:31,863 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T18:52:31,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T18:52:32,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T18:52:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:32,016 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 11883101a4f15c51f8988731ba67e2e0 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-23T18:52:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/7ee248d462b8489cbb48c6aee60e0eba is 1055, key is row0000/info:/1732387951840/Put/seqid=0 2024-11-23T18:52:32,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741843_1019 (size=6008) 2024-11-23T18:52:32,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741843_1019 (size=6008) 2024-11-23T18:52:32,026 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/7ee248d462b8489cbb48c6aee60e0eba 2024-11-23T18:52:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/7ee248d462b8489cbb48c6aee60e0eba as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/7ee248d462b8489cbb48c6aee60e0eba 2024-11-23T18:52:32,040 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/7ee248d462b8489cbb48c6aee60e0eba, entries=1, sequenceid=18, filesize=5.9 K 2024-11-23T18:52:32,041 INFO [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 25ms, sequenceid=18, compaction requested=false 2024-11-23T18:52:32,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 11883101a4f15c51f8988731ba67e2e0: 2024-11-23T18:52:32,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:32,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-23T18:52:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-23T18:52:32,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T18:52:32,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec 2024-11-23T18:52:32,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 187 msec 2024-11-23T18:52:32,322 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:32,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:33,323 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:33,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:34,325 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:34,514 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:35,326 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:35,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:36,328 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:36,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:36,820 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 11883101a4f15c51f8988731ba67e2e0, had cached 0 bytes from a total of 14229 2024-11-23T18:52:37,329 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:37,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:38,330 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:38,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:39,332 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:39,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:40,333 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:40,521 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:41,335 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:41,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T18:52:41,895 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-23T18:52:41,920 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387961901, exclude list is [], retry=0 2024-11-23T18:52:41,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44389,DS-7e3a0a03-3f4c-4bc6-a509-223a3ccc6c1c,DISK] 2024-11-23T18:52:41,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39103,DS-0321bbf7-733e-49ef-8abc-d4b7bdfed69b,DISK] 2024-11-23T18:52:41,924 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841 with entries=3, filesize=1.95 KB; new WAL /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387961901 2024-11-23T18:52:41,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34803:34803),(127.0.0.1/127.0.0.1:35223:35223)] 2024-11-23T18:52:41,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841 is not closed yet, will try archiving it next time 2024-11-23T18:52:41,925 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387941720 to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs/2eda60943ea1%2C40689%2C1732387909610.1732387941720 2024-11-23T18:52:41,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:52:41,925 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:52:41,925 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:52:41,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:41,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:41,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:52:41,926 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:52:41,926 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=941227782, stopped=false 2024-11-23T18:52:41,926 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,37245,1732387909442 2024-11-23T18:52:41,926 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841 not finished, retry = 0 2024-11-23T18:52:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741842_1018 (size=2001) 2024-11-23T18:52:41,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741842_1018 (size=2001) 2024-11-23T18:52:41,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:52:41,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:52:41,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:41,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:41,983 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:52:41,984 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:52:41,984 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:52:41,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:41,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:52:41,984 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,40689,1732387909610' ***** 2024-11-23T18:52:41,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:52:41,985 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:52:41,985 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:52:41,985 INFO [RS:0;2eda60943ea1:40689 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:52:41,985 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:52:41,985 INFO [RS:0;2eda60943ea1:40689 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:52:41,985 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(3091): Received CLOSE for 11883101a4f15c51f8988731ba67e2e0 2024-11-23T18:52:41,986 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,40689,1732387909610 2024-11-23T18:52:41,986 INFO [RS:0;2eda60943ea1:40689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:52:41,986 INFO [RS:0;2eda60943ea1:40689 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:40689. 2024-11-23T18:52:41,986 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 11883101a4f15c51f8988731ba67e2e0, disabling compactions & flushes 2024-11-23T18:52:41,986 DEBUG [RS:0;2eda60943ea1:40689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:52:41,986 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:41,986 DEBUG [RS:0;2eda60943ea1:40689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:41,986 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:41,987 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. after waiting 0 ms 2024-11-23T18:52:41,987 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:41,987 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:52:41,987 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:52:41,987 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:52:41,987 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 11883101a4f15c51f8988731ba67e2e0 1/1 column families, dataSize=1.03 KB heapSize=1.35 KB 2024-11-23T18:52:41,987 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:52:41,987 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:52:41,987 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 11883101a4f15c51f8988731ba67e2e0=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.} 2024-11-23T18:52:41,988 DEBUG [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1351): Waiting on 11883101a4f15c51f8988731ba67e2e0, 1588230740 2024-11-23T18:52:41,988 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:52:41,988 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:52:41,988 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:52:41,988 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:52:41,988 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:52:41,988 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-23T18:52:41,993 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/1f36604d478b4caa873e5bcda50457d2 is 1055, key is row0001/info:/1732387961899/Put/seqid=0 2024-11-23T18:52:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741845_1021 (size=6008) 2024-11-23T18:52:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741845_1021 (size=6008) 2024-11-23T18:52:42,003 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.03 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/1f36604d478b4caa873e5bcda50457d2 2024-11-23T18:52:42,007 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/info/b3e02d788e2b4d7f881b253fed36f1b0 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0./info:regioninfo/1732387911842/Put/seqid=0 2024-11-23T18:52:42,012 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/.tmp/info/1f36604d478b4caa873e5bcda50457d2 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/1f36604d478b4caa873e5bcda50457d2 2024-11-23T18:52:42,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741846_1022 (size=7308) 2024-11-23T18:52:42,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741846_1022 (size=7308) 2024-11-23T18:52:42,014 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/info/b3e02d788e2b4d7f881b253fed36f1b0 2024-11-23T18:52:42,019 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/1f36604d478b4caa873e5bcda50457d2, entries=1, sequenceid=22, filesize=5.9 K 2024-11-23T18:52:42,020 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 33ms, sequenceid=22, compaction requested=true 2024-11-23T18:52:42,021 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780] to archive 2024-11-23T18:52:42,024 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:52:42,027 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/8a7e9a05389349b5a2c79cef6f20825b 2024-11-23T18:52:42,027 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/WALs/2eda60943ea1,40689,1732387909610/2eda60943ea1%2C40689%2C1732387909610.1732387951841 to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs/2eda60943ea1%2C40689%2C1732387909610.1732387951841 2024-11-23T18:52:42,029 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239 to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/a22f44860d104e33826e8ca7079c3239 2024-11-23T18:52:42,030 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780 to hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/info/654b3fbff88c4371bde9253c4e562780 2024-11-23T18:52:42,037 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/ns/9d13ad98cc924eb1a15fb228c161a5b9 is 43, key is default/ns:d/1732387911389/Put/seqid=0 2024-11-23T18:52:42,041 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2eda60943ea1:37245 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-23T18:52:42,041 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8a7e9a05389349b5a2c79cef6f20825b=6008, a22f44860d104e33826e8ca7079c3239=6008, 654b3fbff88c4371bde9253c4e562780=6008] 2024-11-23T18:52:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741847_1023 (size=5153) 2024-11-23T18:52:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741847_1023 (size=5153) 2024-11-23T18:52:42,044 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/ns/9d13ad98cc924eb1a15fb228c161a5b9 2024-11-23T18:52:42,045 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/11883101a4f15c51f8988731ba67e2e0/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-23T18:52:42,046 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:42,046 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 11883101a4f15c51f8988731ba67e2e0: Waiting for close lock at 1732387961986Running coprocessor pre-close hooks at 1732387961986Disabling compacts and flushes for region at 1732387961986Disabling writes for close at 1732387961987 (+1 ms)Obtaining lock to block concurrent updates at 1732387961987Preparing flush snapshotting stores in 11883101a4f15c51f8988731ba67e2e0 at 1732387961987Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0., syncing WAL and waiting on mvcc, flushsize=dataSize=1051, getHeapSize=1368, getOffHeapSize=0, getCellsCount=1 at 1732387961987Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. at 1732387961989 (+2 ms)Flushing 11883101a4f15c51f8988731ba67e2e0/info: creating writer at 1732387961989Flushing 11883101a4f15c51f8988731ba67e2e0/info: appending metadata at 1732387961992 (+3 ms)Flushing 11883101a4f15c51f8988731ba67e2e0/info: closing flushed file at 1732387961992Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37d49a2f: reopening flushed file at 1732387962010 (+18 ms)Finished flush of dataSize ~1.03 KB/1051, heapSize ~1.34 KB/1368, currentSize=0 B/0 for 11883101a4f15c51f8988731ba67e2e0 in 33ms, sequenceid=22, compaction requested=true at 1732387962020 (+10 ms)Writing region close event to WAL at 1732387962042 (+22 ms)Running coprocessor post-close hooks at 1732387962046 (+4 ms)Closed at 1732387962046 2024-11-23T18:52:42,046 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732387911462.11883101a4f15c51f8988731ba67e2e0. 2024-11-23T18:52:42,063 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/table/f5afe60b25db42cb9ef89dc8a173a6d6 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732387911856/Put/seqid=0 2024-11-23T18:52:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741848_1024 (size=5508) 2024-11-23T18:52:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741848_1024 (size=5508) 2024-11-23T18:52:42,069 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/table/f5afe60b25db42cb9ef89dc8a173a6d6 2024-11-23T18:52:42,076 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/info/b3e02d788e2b4d7f881b253fed36f1b0 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/info/b3e02d788e2b4d7f881b253fed36f1b0 2024-11-23T18:52:42,081 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/info/b3e02d788e2b4d7f881b253fed36f1b0, entries=10, sequenceid=11, filesize=7.1 K 2024-11-23T18:52:42,082 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/ns/9d13ad98cc924eb1a15fb228c161a5b9 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/ns/9d13ad98cc924eb1a15fb228c161a5b9 2024-11-23T18:52:42,088 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/ns/9d13ad98cc924eb1a15fb228c161a5b9, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T18:52:42,089 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/.tmp/table/f5afe60b25db42cb9ef89dc8a173a6d6 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/table/f5afe60b25db42cb9ef89dc8a173a6d6 2024-11-23T18:52:42,095 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/table/f5afe60b25db42cb9ef89dc8a173a6d6, entries=2, sequenceid=11, filesize=5.4 K 2024-11-23T18:52:42,096 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=11, compaction requested=false 2024-11-23T18:52:42,101 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T18:52:42,101 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:52:42,101 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:52:42,102 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387961988Running coprocessor pre-close hooks at 1732387961988Disabling compacts and flushes for region at 1732387961988Disabling writes for close at 1732387961988Obtaining lock to block concurrent updates at 1732387961988Preparing flush snapshotting stores in 1588230740 at 1732387961988Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732387961989 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732387961990 (+1 ms)Flushing 1588230740/info: creating writer at 1732387961990Flushing 1588230740/info: appending metadata at 1732387962007 (+17 ms)Flushing 1588230740/info: closing flushed file at 1732387962007Flushing 1588230740/ns: creating writer at 1732387962021 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732387962037 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732387962037Flushing 1588230740/table: creating writer at 1732387962049 (+12 ms)Flushing 1588230740/table: appending metadata at 1732387962062 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732387962062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31bfebc0: reopening flushed file at 1732387962075 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@392f3687: reopening flushed file at 1732387962081 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60f5e8ce: reopening flushed file at 1732387962088 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=11, compaction requested=false at 1732387962096 (+8 ms)Writing region close event to WAL at 1732387962097 (+1 ms)Running coprocessor post-close hooks at 1732387962101 (+4 ms)Closed at 1732387962101 2024-11-23T18:52:42,102 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:52:42,188 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,40689,1732387909610; all regions closed. 2024-11-23T18:52:42,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741834_1010 (size=3306) 2024-11-23T18:52:42,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741834_1010 (size=3306) 2024-11-23T18:52:42,197 DEBUG [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs 2024-11-23T18:52:42,197 INFO [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C40689%2C1732387909610.meta:.meta(num 1732387911281) 2024-11-23T18:52:42,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741844_1020 (size=1252) 2024-11-23T18:52:42,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741844_1020 (size=1252) 2024-11-23T18:52:42,203 DEBUG [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/oldWALs 2024-11-23T18:52:42,203 INFO [RS:0;2eda60943ea1:40689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C40689%2C1732387909610:(num 1732387961901) 2024-11-23T18:52:42,203 DEBUG [RS:0;2eda60943ea1:40689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:42,203 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:52:42,203 INFO [RS:0;2eda60943ea1:40689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:52:42,203 INFO [RS:0;2eda60943ea1:40689 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:52:42,204 INFO [RS:0;2eda60943ea1:40689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:52:42,204 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:52:42,204 INFO [RS:0;2eda60943ea1:40689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40689 2024-11-23T18:52:42,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,40689,1732387909610 2024-11-23T18:52:42,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:52:42,214 INFO [RS:0;2eda60943ea1:40689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:52:42,225 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,40689,1732387909610] 2024-11-23T18:52:42,235 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,40689,1732387909610 already deleted, retry=false 2024-11-23T18:52:42,235 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,40689,1732387909610 expired; onlineServers=0 2024-11-23T18:52:42,235 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,37245,1732387909442' ***** 2024-11-23T18:52:42,235 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:52:42,236 DEBUG [M:0;2eda60943ea1:37245 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:52:42,236 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:52:42,236 DEBUG [M:0;2eda60943ea1:37245 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:52:42,236 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387910421 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387910421,5,FailOnTimeoutGroup] 2024-11-23T18:52:42,236 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387910421 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387910421,5,FailOnTimeoutGroup] 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:52:42,236 DEBUG [M:0;2eda60943ea1:37245 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:52:42,236 INFO [M:0;2eda60943ea1:37245 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:52:42,237 INFO [M:0;2eda60943ea1:37245 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:52:42,237 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:52:42,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:52:42,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:42,246 DEBUG [M:0;2eda60943ea1:37245 {}] zookeeper.ZKUtil(347): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:52:42,246 WARN [M:0;2eda60943ea1:37245 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:52:42,248 INFO [M:0;2eda60943ea1:37245 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/.lastflushedseqids 2024-11-23T18:52:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741849_1025 (size=130) 2024-11-23T18:52:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741849_1025 (size=130) 2024-11-23T18:52:42,255 INFO [M:0;2eda60943ea1:37245 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:52:42,255 INFO [M:0;2eda60943ea1:37245 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:52:42,255 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:52:42,255 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:42,255 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:42,255 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:52:42,255 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:42,255 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.57 KB heapSize=54.96 KB 2024-11-23T18:52:42,269 DEBUG [M:0;2eda60943ea1:37245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/488c16cff6a444e2b2aac77e1edd5dbf is 82, key is hbase:meta,,1/info:regioninfo/1732387911316/Put/seqid=0 2024-11-23T18:52:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741850_1026 (size=5672) 2024-11-23T18:52:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741850_1026 (size=5672) 2024-11-23T18:52:42,275 INFO [M:0;2eda60943ea1:37245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/488c16cff6a444e2b2aac77e1edd5dbf 2024-11-23T18:52:42,294 DEBUG [M:0;2eda60943ea1:37245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40a48dac162340288180622eb5cdf765 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387911862/Put/seqid=0 2024-11-23T18:52:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741851_1027 (size=7820) 2024-11-23T18:52:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741851_1027 (size=7820) 2024-11-23T18:52:42,300 INFO [M:0;2eda60943ea1:37245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40a48dac162340288180622eb5cdf765 2024-11-23T18:52:42,305 INFO [M:0;2eda60943ea1:37245 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 40a48dac162340288180622eb5cdf765 2024-11-23T18:52:42,318 DEBUG [M:0;2eda60943ea1:37245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ecb24f9fb4eb414eb5479fdaadbbfa7a is 69, key is 2eda60943ea1,40689,1732387909610/rs:state/1732387910476/Put/seqid=0 2024-11-23T18:52:42,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741852_1028 (size=5156) 2024-11-23T18:52:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741852_1028 (size=5156) 2024-11-23T18:52:42,324 INFO [M:0;2eda60943ea1:37245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ecb24f9fb4eb414eb5479fdaadbbfa7a 2024-11-23T18:52:42,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:52:42,325 INFO [RS:0;2eda60943ea1:40689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:52:42,325 INFO [RS:0;2eda60943ea1:40689 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,40689,1732387909610; zookeeper connection closed. 2024-11-23T18:52:42,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40689-0x1016909aade0001, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:52:42,325 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d862e50 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d862e50 2024-11-23T18:52:42,326 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T18:52:42,336 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:42,350 DEBUG [M:0;2eda60943ea1:37245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f42a0ca64ef247eb90ddae7077c61bc1 is 52, key is load_balancer_on/state:d/1732387911457/Put/seqid=0 2024-11-23T18:52:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741853_1029 (size=5056) 2024-11-23T18:52:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741853_1029 (size=5056) 2024-11-23T18:52:42,356 INFO [M:0;2eda60943ea1:37245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f42a0ca64ef247eb90ddae7077c61bc1 2024-11-23T18:52:42,362 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/488c16cff6a444e2b2aac77e1edd5dbf as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/488c16cff6a444e2b2aac77e1edd5dbf 2024-11-23T18:52:42,367 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/488c16cff6a444e2b2aac77e1edd5dbf, entries=8, sequenceid=121, filesize=5.5 K 2024-11-23T18:52:42,368 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40a48dac162340288180622eb5cdf765 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40a48dac162340288180622eb5cdf765 2024-11-23T18:52:42,374 INFO [M:0;2eda60943ea1:37245 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 40a48dac162340288180622eb5cdf765 2024-11-23T18:52:42,374 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40a48dac162340288180622eb5cdf765, entries=14, sequenceid=121, filesize=7.6 K 2024-11-23T18:52:42,375 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ecb24f9fb4eb414eb5479fdaadbbfa7a as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ecb24f9fb4eb414eb5479fdaadbbfa7a 2024-11-23T18:52:42,380 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ecb24f9fb4eb414eb5479fdaadbbfa7a, entries=1, sequenceid=121, filesize=5.0 K 2024-11-23T18:52:42,381 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f42a0ca64ef247eb90ddae7077c61bc1 as hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f42a0ca64ef247eb90ddae7077c61bc1 2024-11-23T18:52:42,385 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43611/user/jenkins/test-data/4084cc53-8dc5-e9a3-11a3-1edec96b79e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f42a0ca64ef247eb90ddae7077c61bc1, entries=1, sequenceid=121, filesize=4.9 K 2024-11-23T18:52:42,387 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=121, compaction requested=false 2024-11-23T18:52:42,388 INFO [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:42,388 DEBUG [M:0;2eda60943ea1:37245 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387962255Disabling compacts and flushes for region at 1732387962255Disabling writes for close at 1732387962255Obtaining lock to block concurrent updates at 1732387962255Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732387962255Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44611, getHeapSize=56216, getOffHeapSize=0, getCellsCount=140 at 1732387962255Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732387962256 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732387962256Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732387962269 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732387962269Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732387962280 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732387962294 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732387962294Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732387962305 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732387962318 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732387962318Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732387962330 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732387962349 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732387962349Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d369d08: reopening flushed file at 1732387962361 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26fc5dde: reopening flushed file at 1732387962367 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@172cae35: reopening flushed file at 1732387962374 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@679000b: reopening flushed file at 1732387962380 (+6 ms)Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=121, compaction requested=false at 1732387962387 (+7 ms)Writing region close event to WAL at 1732387962388 (+1 ms)Closed at 1732387962388 2024-11-23T18:52:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39103 is added to blk_1073741830_1006 (size=53008) 2024-11-23T18:52:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44389 is added to blk_1073741830_1006 (size=53008) 2024-11-23T18:52:42,391 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:52:42,391 INFO [M:0;2eda60943ea1:37245 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:52:42,391 INFO [M:0;2eda60943ea1:37245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37245 2024-11-23T18:52:42,391 INFO [M:0;2eda60943ea1:37245 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:52:42,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:52:42,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:52:42,464 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T18:52:42,464 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-23T18:52:42,506 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:52:42,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:42,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:52:42,544 INFO [M:0;2eda60943ea1:37245 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:52:42,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37245-0x1016909aade0000, quorum=127.0.0.1:59128, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:52:42,549 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fcd28da{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:52:42,550 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5fac4361{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:52:42,550 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:52:42,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d453491{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:52:42,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f540c00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,STOPPED} 2024-11-23T18:52:42,554 WARN [BP-1133735507-172.17.0.2-1732387907229 heartbeating to localhost/127.0.0.1:43611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:52:42,554 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:52:42,554 WARN [BP-1133735507-172.17.0.2-1732387907229 heartbeating to localhost/127.0.0.1:43611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1133735507-172.17.0.2-1732387907229 (Datanode Uuid 72fee641-54b7-4b6f-a5ad-ae5bc4ea6750) service to localhost/127.0.0.1:43611 2024-11-23T18:52:42,554 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:52:42,555 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data3/current/BP-1133735507-172.17.0.2-1732387907229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:52:42,556 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data4/current/BP-1133735507-172.17.0.2-1732387907229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:52:42,556 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:52:42,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@628df22e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:52:42,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1db674f6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:52:42,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:52:42,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@457d2735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:52:42,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@440a68df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,STOPPED} 2024-11-23T18:52:42,563 WARN [BP-1133735507-172.17.0.2-1732387907229 heartbeating to localhost/127.0.0.1:43611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:52:42,563 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:52:42,563 WARN [BP-1133735507-172.17.0.2-1732387907229 heartbeating to localhost/127.0.0.1:43611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1133735507-172.17.0.2-1732387907229 (Datanode Uuid 2cbc6015-887b-4c17-870d-44967159fdda) service to localhost/127.0.0.1:43611 2024-11-23T18:52:42,563 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:52:42,564 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data1/current/BP-1133735507-172.17.0.2-1732387907229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:52:42,564 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/cluster_258a2521-4f06-323a-3e4d-5bafdac9ad34/data/data2/current/BP-1133735507-172.17.0.2-1732387907229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:52:42,564 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:52:42,571 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18e25308{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:52:42,571 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5675ef4f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:52:42,571 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:52:42,571 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4475c417{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:52:42,572 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55b966d7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir/,STOPPED} 2024-11-23T18:52:42,578 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:52:42,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T18:52:42,602 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testCompactionRecordDoesntBlockRolling Thread=97 (was 102), OpenFileDescriptor=428 (was 434), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=86 (was 106), ProcessCount=11 (was 11), AvailableMemoryMB=8736 (was 8893) 2024-11-23T18:52:42,607 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRolling Thread=97, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=86, ProcessCount=11, AvailableMemoryMB=8736 2024-11-23T18:52:42,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:52:42,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.log.dir so I do NOT create it in target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a03aa9f-a1f9-5277-2e02-af5034925131/hadoop.tmp.dir so I do NOT create it in target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4, deleteOnExit=true 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/test.cache.data in system properties and HBase conf 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:52:42,608 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:52:42,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:52:42,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:52:43,077 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:52:43,081 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:52:43,082 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:52:43,082 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:52:43,082 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:52:43,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:52:43,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@758d42f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:52:43,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e9b2284{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:52:43,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ace8d8a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/java.io.tmpdir/jetty-localhost-44643-hadoop-hdfs-3_4_1-tests_jar-_-any-15460365696047797472/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:52:43,176 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22ac2cd9{HTTP/1.1, (http/1.1)}{localhost:44643} 2024-11-23T18:52:43,176 INFO [Time-limited test {}] server.Server(415): Started @157326ms 2024-11-23T18:52:43,336 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:43,446 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:52:43,450 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:52:43,452 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:52:43,452 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:52:43,452 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:52:43,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@325aed0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:52:43,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b6547b7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:52:43,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:43,549 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@483a949e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/java.io.tmpdir/jetty-localhost-43067-hadoop-hdfs-3_4_1-tests_jar-_-any-9472508688354623178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:52:43,549 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@76e3af92{HTTP/1.1, (http/1.1)}{localhost:43067} 2024-11-23T18:52:43,550 INFO [Time-limited test {}] server.Server(415): Started @157699ms 2024-11-23T18:52:43,551 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:52:43,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:52:43,580 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:52:43,581 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:52:43,581 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:52:43,581 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:52:43,582 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@88cc783{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:52:43,582 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@dc32d49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:52:43,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e9a7ffb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/java.io.tmpdir/jetty-localhost-38163-hadoop-hdfs-3_4_1-tests_jar-_-any-11972189258505601942/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:52:43,676 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b485209{HTTP/1.1, (http/1.1)}{localhost:38163} 2024-11-23T18:52:43,676 INFO [Time-limited test {}] server.Server(415): Started @157825ms 2024-11-23T18:52:43,677 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:52:44,337 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:44,525 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:44,816 WARN [Thread-1103 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data1/current/BP-1758811751-172.17.0.2-1732387962631/current, will proceed with Du for space computation calculation, 2024-11-23T18:52:44,816 WARN [Thread-1104 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data2/current/BP-1758811751-172.17.0.2-1732387962631/current, will proceed with Du for space computation calculation, 2024-11-23T18:52:44,832 WARN [Thread-1067 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:52:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63a37f92ed2f782e with lease ID 0x22644ca016dabd6f: Processing first storage report for DS-e098b697-be36-48fe-997e-a766f9421635 from datanode DatanodeRegistration(127.0.0.1:41159, datanodeUuid=d48fff1c-97df-4257-9676-5652a32cabe0, infoPort=44105, infoSecurePort=0, ipcPort=40709, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631) 2024-11-23T18:52:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63a37f92ed2f782e with lease ID 0x22644ca016dabd6f: from storage DS-e098b697-be36-48fe-997e-a766f9421635 node DatanodeRegistration(127.0.0.1:41159, datanodeUuid=d48fff1c-97df-4257-9676-5652a32cabe0, infoPort=44105, infoSecurePort=0, ipcPort=40709, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:52:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63a37f92ed2f782e with lease ID 0x22644ca016dabd6f: Processing first storage report for DS-94314bce-dcb3-4f3a-9bae-150dd09dfd6b from datanode DatanodeRegistration(127.0.0.1:41159, datanodeUuid=d48fff1c-97df-4257-9676-5652a32cabe0, infoPort=44105, infoSecurePort=0, ipcPort=40709, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631) 2024-11-23T18:52:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63a37f92ed2f782e with lease ID 0x22644ca016dabd6f: from storage DS-94314bce-dcb3-4f3a-9bae-150dd09dfd6b node DatanodeRegistration(127.0.0.1:41159, datanodeUuid=d48fff1c-97df-4257-9676-5652a32cabe0, infoPort=44105, infoSecurePort=0, ipcPort=40709, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:52:44,940 WARN [Thread-1114 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data3/current/BP-1758811751-172.17.0.2-1732387962631/current, will proceed with Du for space computation calculation, 2024-11-23T18:52:44,940 WARN [Thread-1115 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data4/current/BP-1758811751-172.17.0.2-1732387962631/current, will proceed with Du for space computation calculation, 2024-11-23T18:52:44,961 WARN [Thread-1090 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:52:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82bb8846a8f8fc31 with lease ID 0x22644ca016dabd70: Processing first storage report for DS-35dd9149-e2ad-4314-ba5c-dcebffabe180 from datanode DatanodeRegistration(127.0.0.1:46545, datanodeUuid=69490fd6-ea36-4691-ab17-44fd903fda7c, infoPort=33185, infoSecurePort=0, ipcPort=36955, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631) 2024-11-23T18:52:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82bb8846a8f8fc31 with lease ID 0x22644ca016dabd70: from storage DS-35dd9149-e2ad-4314-ba5c-dcebffabe180 node DatanodeRegistration(127.0.0.1:46545, datanodeUuid=69490fd6-ea36-4691-ab17-44fd903fda7c, infoPort=33185, infoSecurePort=0, ipcPort=36955, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:52:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82bb8846a8f8fc31 with lease ID 0x22644ca016dabd70: Processing first storage report for DS-ec86d95f-9d84-477d-912e-9d0e156b1bc4 from datanode DatanodeRegistration(127.0.0.1:46545, datanodeUuid=69490fd6-ea36-4691-ab17-44fd903fda7c, infoPort=33185, infoSecurePort=0, ipcPort=36955, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631) 2024-11-23T18:52:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82bb8846a8f8fc31 with lease ID 0x22644ca016dabd70: from storage DS-ec86d95f-9d84-477d-912e-9d0e156b1bc4 node DatanodeRegistration(127.0.0.1:46545, datanodeUuid=69490fd6-ea36-4691-ab17-44fd903fda7c, infoPort=33185, infoSecurePort=0, ipcPort=36955, storageInfo=lv=-57;cid=testClusterID;nsid=1849029494;c=1732387962631), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:52:45,015 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20 2024-11-23T18:52:45,020 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/zookeeper_0, clientPort=56959, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:52:45,021 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56959 2024-11-23T18:52:45,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,022 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:52:45,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:52:45,030 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40 with version=8 2024-11-23T18:52:45,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase-staging 2024-11-23T18:52:45,032 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:52:45,032 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:52:45,033 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34751 2024-11-23T18:52:45,035 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34751 connecting to ZooKeeper ensemble=127.0.0.1:56959 2024-11-23T18:52:45,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:347510x0, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:52:45,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34751-0x101690a84060000 connected 2024-11-23T18:52:45,183 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:52:45,186 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40, hbase.cluster.distributed=false 2024-11-23T18:52:45,187 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:52:45,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34751 2024-11-23T18:52:45,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34751 2024-11-23T18:52:45,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34751 2024-11-23T18:52:45,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34751 2024-11-23T18:52:45,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34751 2024-11-23T18:52:45,202 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:52:45,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:52:45,203 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46533 2024-11-23T18:52:45,205 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46533 connecting to ZooKeeper ensemble=127.0.0.1:56959 2024-11-23T18:52:45,205 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,207 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465330x0, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:52:45,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46533-0x101690a84060001 connected 2024-11-23T18:52:45,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:52:45,215 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:52:45,216 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:52:45,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:52:45,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:52:45,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46533 2024-11-23T18:52:45,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46533 2024-11-23T18:52:45,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46533 2024-11-23T18:52:45,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46533 2024-11-23T18:52:45,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46533 2024-11-23T18:52:45,230 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:34751 2024-11-23T18:52:45,230 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:52:45,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:52:45,236 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:52:45,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,246 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:52:45,247 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,34751,1732387965032 from backup master directory 2024-11-23T18:52:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:52:45,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:52:45,256 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:52:45,257 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,264 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/hbase.id] with ID: b366b1ef-e9b2-4793-86a5-b1afbdf687c8 2024-11-23T18:52:45,264 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/.tmp/hbase.id 2024-11-23T18:52:45,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:52:45,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:52:45,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/.tmp/hbase.id]:[hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/hbase.id] 2024-11-23T18:52:45,284 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:45,284 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:52:45,285 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-23T18:52:45,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:52:45,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:52:45,305 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:52:45,306 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:52:45,306 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:52:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:52:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:52:45,314 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store 2024-11-23T18:52:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:52:45,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:52:45,320 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:45,320 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:52:45,320 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387965320Disabling compacts and flushes for region at 1732387965320Disabling writes for close at 1732387965320Writing region close event to WAL at 1732387965320Closed at 1732387965320 2024-11-23T18:52:45,321 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/.initializing 2024-11-23T18:52:45,321 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/WALs/2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,322 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:52:45,324 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C34751%2C1732387965032, suffix=, logDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/WALs/2eda60943ea1,34751,1732387965032, archiveDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/oldWALs, maxLogs=10 2024-11-23T18:52:45,336 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/WALs/2eda60943ea1,34751,1732387965032/2eda60943ea1%2C34751%2C1732387965032.1732387965324, exclude list is [], retry=0 2024-11-23T18:52:45,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41159,DS-e098b697-be36-48fe-997e-a766f9421635,DISK] 2024-11-23T18:52:45,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46545,DS-35dd9149-e2ad-4314-ba5c-dcebffabe180,DISK] 2024-11-23T18:52:45,339 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:45,341 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/WALs/2eda60943ea1,34751,1732387965032/2eda60943ea1%2C34751%2C1732387965032.1732387965324 2024-11-23T18:52:45,342 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:33185:33185)] 2024-11-23T18:52:45,342 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:52:45,342 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:45,342 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,342 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:52:45,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:45,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:52:45,347 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:52:45,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:52:45,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:52:45,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:52:45,351 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:52:45,351 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,352 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,352 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,353 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,353 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,354 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:52:45,355 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:52:45,357 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:52:45,357 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=753136, jitterRate=-0.04233811795711517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:52:45,358 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387965342Initializing all the Stores at 1732387965343 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387965343Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387965343Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387965343Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387965343Cleaning up temporary data from old regions at 1732387965353 (+10 ms)Region opened successfully at 1732387965358 (+5 ms) 2024-11-23T18:52:45,358 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:52:45,361 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acb54c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:52:45,362 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T18:52:45,363 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:52:45,365 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:52:45,366 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:52:45,372 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:52:45,372 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:52:45,373 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:52:45,382 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:52:45,383 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:52:45,384 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:52:45,393 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:52:45,394 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:52:45,404 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:52:45,406 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:52:45,414 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:52:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:52:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:52:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,425 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,34751,1732387965032, sessionid=0x101690a84060000, setting cluster-up flag (Was=false) 2024-11-23T18:52:45,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,477 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:52:45,479 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:45,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:45,530 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:52:45,532 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,34751,1732387965032 2024-11-23T18:52:45,534 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:52:45,535 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:52:45,536 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:52:45,536 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:52:45,536 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,34751,1732387965032 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:52:45,539 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387995541 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:52:45,541 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:52:45,541 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:52:45,541 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:52:45,542 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,542 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:52:45,542 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:52:45,542 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:52:45,543 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387965543,5,FailOnTimeoutGroup] 2024-11-23T18:52:45,543 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387965543,5,FailOnTimeoutGroup] 2024-11-23T18:52:45,543 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,543 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,543 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:52:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:52:45,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:52:45,554 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:52:45,554 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40 2024-11-23T18:52:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:52:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:52:45,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:45,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:52:45,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:52:45,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:45,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:52:45,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:52:45,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:45,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:52:45,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:52:45,567 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:45,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:52:45,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:52:45,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:45,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:45,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:52:45,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740 2024-11-23T18:52:45,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740 2024-11-23T18:52:45,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:52:45,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:52:45,572 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:52:45,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:52:45,575 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:52:45,576 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733843, jitterRate=-0.06687118113040924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:52:45,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387965561Initializing all the Stores at 1732387965562 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387965562Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387965562Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387965562Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387965562Cleaning up temporary data from old regions at 1732387965572 (+10 ms)Region opened successfully at 1732387965576 (+4 ms) 2024-11-23T18:52:45,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:52:45,576 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:52:45,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:52:45,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:52:45,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:52:45,577 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:52:45,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387965576Disabling compacts and flushes for region at 1732387965576Disabling writes for close at 1732387965577 (+1 ms)Writing region close event to WAL at 1732387965577Closed at 1732387965577 2024-11-23T18:52:45,578 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:52:45,578 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:52:45,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:52:45,579 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:52:45,580 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:52:45,622 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(746): ClusterId : b366b1ef-e9b2-4793-86a5-b1afbdf687c8 2024-11-23T18:52:45,622 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:52:45,637 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:52:45,637 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:52:45,648 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:52:45,649 DEBUG [RS:0;2eda60943ea1:46533 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6b7edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:52:45,666 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:46533 2024-11-23T18:52:45,666 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:52:45,666 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:52:45,666 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:52:45,667 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,34751,1732387965032 with port=46533, startcode=1732387965202 2024-11-23T18:52:45,667 DEBUG [RS:0;2eda60943ea1:46533 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:52:45,668 INFO [HMaster-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43071, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:52:45,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34751 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34751 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,670 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40 2024-11-23T18:52:45,670 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39149 2024-11-23T18:52:45,670 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:52:45,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:52:45,678 DEBUG [RS:0;2eda60943ea1:46533 {}] zookeeper.ZKUtil(111): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,678 WARN [RS:0;2eda60943ea1:46533 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:52:45,678 INFO [RS:0;2eda60943ea1:46533 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:52:45,678 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,678 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,46533,1732387965202] 2024-11-23T18:52:45,681 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:52:45,683 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:52:45,683 INFO [RS:0;2eda60943ea1:46533 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:52:45,683 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,683 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:52:45,684 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:52:45,684 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,684 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:52:45,685 DEBUG [RS:0;2eda60943ea1:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,686 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,46533,1732387965202-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:52:45,702 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:52:45,702 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,46533,1732387965202-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,703 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,703 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.Replication(171): 2eda60943ea1,46533,1732387965202 started 2024-11-23T18:52:45,715 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:45,715 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,46533,1732387965202, RpcServer on 2eda60943ea1/172.17.0.2:46533, sessionid=0x101690a84060001 2024-11-23T18:52:45,715 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:52:45,715 DEBUG [RS:0;2eda60943ea1:46533 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,715 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,46533,1732387965202' 2024-11-23T18:52:45,715 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,46533,1732387965202' 2024-11-23T18:52:45,716 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:52:45,717 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:52:45,717 DEBUG [RS:0;2eda60943ea1:46533 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:52:45,717 INFO [RS:0;2eda60943ea1:46533 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:52:45,717 INFO [RS:0;2eda60943ea1:46533 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:52:45,730 WARN [2eda60943ea1:34751 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:52:45,818 INFO [RS:0;2eda60943ea1:46533 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:52:45,820 INFO [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C46533%2C1732387965202, suffix=, logDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202, archiveDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs, maxLogs=32 2024-11-23T18:52:45,834 DEBUG [RS:0;2eda60943ea1:46533 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732387965820, exclude list is [], retry=0 2024-11-23T18:52:45,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41159,DS-e098b697-be36-48fe-997e-a766f9421635,DISK] 2024-11-23T18:52:45,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46545,DS-35dd9149-e2ad-4314-ba5c-dcebffabe180,DISK] 2024-11-23T18:52:45,839 INFO [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732387965820 2024-11-23T18:52:45,839 DEBUG [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:33185:33185)] 2024-11-23T18:52:45,981 DEBUG [2eda60943ea1:34751 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T18:52:45,981 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:45,982 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,46533,1732387965202, state=OPENING 2024-11-23T18:52:45,993 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:52:46,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:46,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:52:46,004 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:52:46,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:52:46,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:52:46,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,46533,1732387965202}] 2024-11-23T18:52:46,158 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:52:46,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:52:46,166 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:52:46,167 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:52:46,167 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:52:46,171 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C46533%2C1732387965202.meta, suffix=.meta, logDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202, archiveDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs, maxLogs=32 2024-11-23T18:52:46,185 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.meta.1732387966171.meta, exclude list is [], retry=0 2024-11-23T18:52:46,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46545,DS-35dd9149-e2ad-4314-ba5c-dcebffabe180,DISK] 2024-11-23T18:52:46,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41159,DS-e098b697-be36-48fe-997e-a766f9421635,DISK] 2024-11-23T18:52:46,189 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.meta.1732387966171.meta 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33185:33185),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:52:46,190 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:52:46,190 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:52:46,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:52:46,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:52:46,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:46,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:52:46,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:52:46,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:46,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:52:46,195 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:52:46,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:46,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:52:46,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:52:46,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:52:46,197 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:52:46,198 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740 2024-11-23T18:52:46,199 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740 2024-11-23T18:52:46,201 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:52:46,201 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:52:46,201 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:52:46,202 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:52:46,203 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=688335, jitterRate=-0.12473709881305695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:52:46,203 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:52:46,204 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387966191Writing region info on filesystem at 1732387966191Initializing all the Stores at 1732387966191Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387966191Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387966192 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387966192Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387966192Cleaning up temporary data from old regions at 1732387966201 (+9 ms)Running coprocessor post-open hooks at 1732387966203 (+2 ms)Region opened successfully at 1732387966204 (+1 ms) 2024-11-23T18:52:46,205 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387966157 2024-11-23T18:52:46,207 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:52:46,207 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:52:46,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:46,208 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,46533,1732387965202, state=OPEN 2024-11-23T18:52:46,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:52:46,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:52:46,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:52:46,246 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:46,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:52:46,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:52:46,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,46533,1732387965202 in 242 msec 2024-11-23T18:52:46,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:52:46,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 674 msec 2024-11-23T18:52:46,256 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:52:46,256 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:52:46,257 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:52:46,257 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,46533,1732387965202, seqNum=-1] 2024-11-23T18:52:46,258 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:52:46,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:52:46,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 729 msec 2024-11-23T18:52:46,265 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387966265, completionTime=-1 2024-11-23T18:52:46,265 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T18:52:46,265 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:52:46,266 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732388026266 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732388086267 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:34751, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,269 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:52:46,270 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.013sec 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:52:46,271 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:52:46,273 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:52:46,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:52:46,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,34751,1732387965032-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:52:46,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bb6147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:52:46,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,34751,-1 for getting cluster id 2024-11-23T18:52:46,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:52:46,325 DEBUG [HMaster-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b366b1ef-e9b2-4793-86a5-b1afbdf687c8' 2024-11-23T18:52:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:52:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b366b1ef-e9b2-4793-86a5-b1afbdf687c8" 2024-11-23T18:52:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a64bfc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:52:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,34751,-1] 2024-11-23T18:52:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:52:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:52:46,330 INFO [HMaster-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:52:46,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc1a763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:52:46,333 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:52:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,46533,1732387965202, seqNum=-1] 2024-11-23T18:52:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:52:46,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:52:46,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,34751,1732387965032 2024-11-23T18:52:46,337 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:52:46,339 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-23T18:52:46,339 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:52:46,339 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,34751,1732387965032 2024-11-23T18:52:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59fcd3f0 2024-11-23T18:52:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:52:46,342 INFO [HMaster-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:52:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-23T18:52:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-23T18:52:46,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:52:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-23T18:52:46,346 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:52:46,346 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-23T18:52:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:52:46,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:52:46,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741835_1011 (size=381) 2024-11-23T18:52:46,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741835_1011 (size=381) 2024-11-23T18:52:46,355 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a23b45b30de756a61d82ac4bd938eb32, NAME => 'TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40 2024-11-23T18:52:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741836_1012 (size=64) 2024-11-23T18:52:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741836_1012 (size=64) 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing a23b45b30de756a61d82ac4bd938eb32, disabling compactions & flushes 2024-11-23T18:52:46,362 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. after waiting 0 ms 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,362 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,362 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for a23b45b30de756a61d82ac4bd938eb32: Waiting for close lock at 1732387966362Disabling compacts and flushes for region at 1732387966362Disabling writes for close at 1732387966362Writing region close event to WAL at 1732387966362Closed at 1732387966362 2024-11-23T18:52:46,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:52:46,364 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732387966364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387966364"}]},"ts":"1732387966364"} 2024-11-23T18:52:46,366 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:52:46,367 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:52:46,367 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387966367"}]},"ts":"1732387966367"} 2024-11-23T18:52:46,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-23T18:52:46,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, ASSIGN}] 2024-11-23T18:52:46,370 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, ASSIGN 2024-11-23T18:52:46,370 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, ASSIGN; state=OFFLINE, location=2eda60943ea1,46533,1732387965202; forceNewPlan=false, retain=false 2024-11-23T18:52:46,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a23b45b30de756a61d82ac4bd938eb32, regionState=OPENING, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:46,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:46,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, ASSIGN because future has completed 2024-11-23T18:52:46,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202}] 2024-11-23T18:52:46,693 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,693 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a23b45b30de756a61d82ac4bd938eb32, NAME => 'TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:52:46,694 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,694 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:52:46,694 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,694 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,697 INFO [StoreOpener-a23b45b30de756a61d82ac4bd938eb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,699 INFO [StoreOpener-a23b45b30de756a61d82ac4bd938eb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23b45b30de756a61d82ac4bd938eb32 columnFamilyName info 2024-11-23T18:52:46,699 DEBUG [StoreOpener-a23b45b30de756a61d82ac4bd938eb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:52:46,700 INFO [StoreOpener-a23b45b30de756a61d82ac4bd938eb32-1 {}] regionserver.HStore(327): Store=a23b45b30de756a61d82ac4bd938eb32/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:52:46,700 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,701 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,702 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,702 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,702 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,706 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,709 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:52:46,710 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a23b45b30de756a61d82ac4bd938eb32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862695, jitterRate=0.09697428345680237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:52:46,710 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:46,711 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a23b45b30de756a61d82ac4bd938eb32: Running coprocessor pre-open hook at 1732387966694Writing region info on filesystem at 1732387966694Initializing all the Stores at 1732387966696 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387966696Cleaning up temporary data from old regions at 1732387966703 (+7 ms)Running coprocessor post-open hooks at 1732387966710 (+7 ms)Region opened successfully at 1732387966711 (+1 ms) 2024-11-23T18:52:46,713 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., pid=6, masterSystemTime=1732387966683 2024-11-23T18:52:46,715 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,716 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:46,717 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a23b45b30de756a61d82ac4bd938eb32, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:52:46,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 because future has completed 2024-11-23T18:52:46,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:52:46,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 in 192 msec 2024-11-23T18:52:46,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:52:46,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, ASSIGN in 355 msec 2024-11-23T18:52:46,726 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:52:46,727 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387966726"}]},"ts":"1732387966726"} 2024-11-23T18:52:46,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-23T18:52:46,730 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:52:46,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 387 msec 2024-11-23T18:52:47,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,066 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,066 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,071 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,340 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:47,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:47,576 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:52:47,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,596 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,596 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,596 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:47,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:52:48,341 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:48,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:49,343 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:49,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:50,344 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:50,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:51,345 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:51,346 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 after 68090ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:52:51,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:51,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta after 68082ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:52:51,682 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:52:51,684 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-23T18:52:52,347 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:52,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:52:52,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T18:52:52,466 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:52:52,466 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T18:52:52,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T18:52:52,467 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T18:52:52,469 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-23T18:52:52,469 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-23T18:52:52,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:53,349 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:53,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:54,350 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:54,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:55,352 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:55,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:56,353 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:52:56,444 INFO [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-23T18:52:56,444 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-23T18:52:56,451 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-23T18:52:56,451 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2] 2024-11-23T18:52:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:56,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:52:56,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/9b13d0669afb4a898ad3d15cd0b0eb5d is 1055, key is row0001/info:/1732387976456/Put/seqid=0 2024-11-23T18:52:56,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741837_1013 (size=13386) 2024-11-23T18:52:56,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741837_1013 (size=13386) 2024-11-23T18:52:56,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/9b13d0669afb4a898ad3d15cd0b0eb5d 2024-11-23T18:52:56,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/9b13d0669afb4a898ad3d15cd0b0eb5d as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d 2024-11-23T18:52:56,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d, entries=8, sequenceid=12, filesize=13.1 K 2024-11-23T18:52:56,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=18.47 KB/18918 for a23b45b30de756a61d82ac4bd938eb32 in 41ms, sequenceid=12, compaction requested=false 2024-11-23T18:52:56,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:52:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:56,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=20.53 KB heapSize=22.28 KB 2024-11-23T18:52:56,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/f015526e4c964158953ac165762795aa is 1055, key is row0009/info:/1732387976472/Put/seqid=0 2024-11-23T18:52:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741838_1014 (size=26030) 2024-11-23T18:52:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741838_1014 (size=26030) 2024-11-23T18:52:56,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.53 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/f015526e4c964158953ac165762795aa 2024-11-23T18:52:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/f015526e4c964158953ac165762795aa as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa 2024-11-23T18:52:56,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa, entries=20, sequenceid=35, filesize=25.4 K 2024-11-23T18:52:56,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~20.53 KB/21020, heapSize ~22.27 KB/22800, currentSize=4.11 KB/4204 for a23b45b30de756a61d82ac4bd938eb32 in 24ms, sequenceid=35, compaction requested=false 2024-11-23T18:52:56,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:52:56,539 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.5 K, sizeToCheck=16.0 K 2024-11-23T18:52:56,539 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:56,539 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa because midkey is the same as first or last row 2024-11-23T18:52:56,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:57,355 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:57,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:58,356 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:58,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:52:58,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:58,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/b5d571b1c254428c8d1c62ab444fbba1 is 1055, key is row0029/info:/1732387976516/Put/seqid=0 2024-11-23T18:52:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741839_1015 (size=13386) 2024-11-23T18:52:58,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741839_1015 (size=13386) 2024-11-23T18:52:58,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/b5d571b1c254428c8d1c62ab444fbba1 2024-11-23T18:52:58,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/b5d571b1c254428c8d1c62ab444fbba1 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1 2024-11-23T18:52:58,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1, entries=8, sequenceid=46, filesize=13.1 K 2024-11-23T18:52:58,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=11.29 KB/11561 for a23b45b30de756a61d82ac4bd938eb32 in 27ms, sequenceid=46, compaction requested=true 2024-11-23T18:52:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:52:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=51.6 K, sizeToCheck=16.0 K 2024-11-23T18:52:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa because midkey is the same as first or last row 2024-11-23T18:52:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:52:58,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23b45b30de756a61d82ac4bd938eb32:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:52:58,566 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:52:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:52:58,567 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 52802 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:52:58,567 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): a23b45b30de756a61d82ac4bd938eb32/info is initiating minor compaction (all files) 2024-11-23T18:52:58,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-23T18:52:58,568 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23b45b30de756a61d82ac4bd938eb32/info in TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:52:58,568 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp, totalSize=51.6 K 2024-11-23T18:52:58,569 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b13d0669afb4a898ad3d15cd0b0eb5d, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732387976456 2024-11-23T18:52:58,570 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting f015526e4c964158953ac165762795aa, keycount=20, bloomtype=ROW, size=25.4 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732387976472 2024-11-23T18:52:58,570 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d571b1c254428c8d1c62ab444fbba1, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732387976516 2024-11-23T18:52:58,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/ebba63b1a9e2433186768fa34ee81ce1 is 1055, key is row0037/info:/1732387978538/Put/seqid=0 2024-11-23T18:52:58,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741840_1016 (size=18662) 2024-11-23T18:52:58,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741840_1016 (size=18662) 2024-11-23T18:52:58,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/ebba63b1a9e2433186768fa34ee81ce1 2024-11-23T18:52:58,591 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23b45b30de756a61d82ac4bd938eb32#info#compaction#35 average throughput is 12.03 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:52:58,592 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/56524828ceeb4bddac1b5740982943a6 is 1055, key is row0001/info:/1732387976456/Put/seqid=0 2024-11-23T18:52:58,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/ebba63b1a9e2433186768fa34ee81ce1 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1 2024-11-23T18:52:58,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741841_1017 (size=43001) 2024-11-23T18:52:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741841_1017 (size=43001) 2024-11-23T18:52:58,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1, entries=13, sequenceid=62, filesize=18.2 K 2024-11-23T18:52:58,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=15.40 KB/15765 for a23b45b30de756a61d82ac4bd938eb32 in 36ms, sequenceid=62, compaction requested=false 2024-11-23T18:52:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:52:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.8 K, sizeToCheck=16.0 K 2024-11-23T18:52:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa because midkey is the same as first or last row 2024-11-23T18:52:58,605 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/56524828ceeb4bddac1b5740982943a6 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 2024-11-23T18:52:58,612 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23b45b30de756a61d82ac4bd938eb32/info of a23b45b30de756a61d82ac4bd938eb32 into 56524828ceeb4bddac1b5740982943a6(size=42.0 K), total size for store is 60.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:52:58,612 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:52:58,613 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., storeName=a23b45b30de756a61d82ac4bd938eb32/info, priority=13, startTime=1732387978565; duration=0sec 2024-11-23T18:52:58,613 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 because midkey is the same as first or last row 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 because midkey is the same as first or last row 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.2 K, sizeToCheck=16.0 K 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 because midkey is the same as first or last row 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:52:58,614 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23b45b30de756a61d82ac4bd938eb32:info 2024-11-23T18:52:59,357 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:52:59,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:00,359 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:00,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:00,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=16.42 KB heapSize=17.88 KB 2024-11-23T18:53:00,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/5601e81a619a4adca4c9a54bca9c1e86 is 1055, key is row0050/info:/1732387978569/Put/seqid=0 2024-11-23T18:53:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741842_1018 (size=21822) 2024-11-23T18:53:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741842_1018 (size=21822) 2024-11-23T18:53:00,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.42 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/5601e81a619a4adca4c9a54bca9c1e86 2024-11-23T18:53:00,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/5601e81a619a4adca4c9a54bca9c1e86 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86 2024-11-23T18:53:00,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86, entries=16, sequenceid=82, filesize=21.3 K 2024-11-23T18:53:00,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T18:53:00,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.42 KB/16816, heapSize ~17.86 KB/18288, currentSize=13.34 KB/13663 for a23b45b30de756a61d82ac4bd938eb32 in 36ms, sequenceid=82, compaction requested=true 2024-11-23T18:53:00,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:53:00,644 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.5 K, sizeToCheck=16.0 K 2024-11-23T18:53:00,644 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:53:00,645 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 because midkey is the same as first or last row 2024-11-23T18:53:00,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23b45b30de756a61d82ac4bd938eb32:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:00,645 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:00,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:00,646 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 83485 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:00,646 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): a23b45b30de756a61d82ac4bd938eb32/info is initiating minor compaction (all files) 2024-11-23T18:53:00,646 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23b45b30de756a61d82ac4bd938eb32/info in TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:53:00,646 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp, totalSize=81.5 K 2024-11-23T18:53:00,647 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56524828ceeb4bddac1b5740982943a6, keycount=36, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732387976456 2024-11-23T18:53:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:54148 deadline: 1732387990643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:00,647 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebba63b1a9e2433186768fa34ee81ce1, keycount=13, bloomtype=ROW, size=18.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732387978538 2024-11-23T18:53:00,648 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5601e81a619a4adca4c9a54bca9c1e86, keycount=16, bloomtype=ROW, size=21.3 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732387978569 2024-11-23T18:53:00,659 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23b45b30de756a61d82ac4bd938eb32#info#compaction#37 average throughput is 32.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:00,660 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/582dadf957b5455e820e7df589ca7352 is 1055, key is row0001/info:/1732387976456/Put/seqid=0 2024-11-23T18:53:00,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741843_1019 (size=73753) 2024-11-23T18:53:00,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741843_1019 (size=73753) 2024-11-23T18:53:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T18:53:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T18:53:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 because the exception is null or not the one we care about 2024-11-23T18:53:01,080 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/582dadf957b5455e820e7df589ca7352 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352 2024-11-23T18:53:01,088 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23b45b30de756a61d82ac4bd938eb32/info of a23b45b30de756a61d82ac4bd938eb32 into 582dadf957b5455e820e7df589ca7352(size=72.0 K), total size for store is 72.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:01,088 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23b45b30de756a61d82ac4bd938eb32: 2024-11-23T18:53:01,088 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., storeName=a23b45b30de756a61d82ac4bd938eb32/info, priority=13, startTime=1732387980645; duration=0sec 2024-11-23T18:53:01,088 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.0 K, sizeToCheck=16.0 K 2024-11-23T18:53:01,088 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:53:01,088 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.0 K, sizeToCheck=16.0 K 2024-11-23T18:53:01,088 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:53:01,089 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.0 K, sizeToCheck=16.0 K 2024-11-23T18:53:01,089 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-23T18:53:01,089 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,089 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,090 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23b45b30de756a61d82ac4bd938eb32:info 2024-11-23T18:53:01,091 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34751 {}] assignment.AssignmentManager(1363): Split request from 2eda60943ea1,46533,1732387965202, parent={ENCODED => a23b45b30de756a61d82ac4bd938eb32, NAME => 'TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.', STARTKEY => '', ENDKEY => ''}, splitKey=row0064 2024-11-23T18:53:01,095 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34751 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,099 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34751 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a23b45b30de756a61d82ac4bd938eb32, daughterA=e4579d7a4e55da8b5d463e229943fc06, daughterB=9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a23b45b30de756a61d82ac4bd938eb32, daughterA=e4579d7a4e55da8b5d463e229943fc06, daughterB=9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a23b45b30de756a61d82ac4bd938eb32, daughterA=e4579d7a4e55da8b5d463e229943fc06, daughterB=9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a23b45b30de756a61d82ac4bd938eb32, daughterA=e4579d7a4e55da8b5d463e229943fc06, daughterB=9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, UNASSIGN}] 2024-11-23T18:53:01,109 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, UNASSIGN 2024-11-23T18:53:01,110 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a23b45b30de756a61d82ac4bd938eb32, regionState=CLOSING, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, UNASSIGN because future has completed 2024-11-23T18:53:01,113 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-23T18:53:01,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202}] 2024-11-23T18:53:01,274 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,274 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-23T18:53:01,275 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing a23b45b30de756a61d82ac4bd938eb32, disabling compactions & flushes 2024-11-23T18:53:01,276 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:53:01,276 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:53:01,276 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. after waiting 0 ms 2024-11-23T18:53:01,276 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:53:01,276 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing a23b45b30de756a61d82ac4bd938eb32 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-23T18:53:01,285 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/d533def941dc4b54abb6d295c753d428 is 1055, key is row0066/info:/1732387980611/Put/seqid=0 2024-11-23T18:53:01,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741844_1020 (size=18662) 2024-11-23T18:53:01,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741844_1020 (size=18662) 2024-11-23T18:53:01,291 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/d533def941dc4b54abb6d295c753d428 2024-11-23T18:53:01,299 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/.tmp/info/d533def941dc4b54abb6d295c753d428 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/d533def941dc4b54abb6d295c753d428 2024-11-23T18:53:01,304 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/d533def941dc4b54abb6d295c753d428, entries=13, sequenceid=99, filesize=18.2 K 2024-11-23T18:53:01,305 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=0 B/0 for a23b45b30de756a61d82ac4bd938eb32 in 29ms, sequenceid=99, compaction requested=false 2024-11-23T18:53:01,306 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86] to archive 2024-11-23T18:53:01,307 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:53:01,309 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/9b13d0669afb4a898ad3d15cd0b0eb5d 2024-11-23T18:53:01,310 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/f015526e4c964158953ac165762795aa 2024-11-23T18:53:01,312 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/56524828ceeb4bddac1b5740982943a6 2024-11-23T18:53:01,313 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/b5d571b1c254428c8d1c62ab444fbba1 2024-11-23T18:53:01,314 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/ebba63b1a9e2433186768fa34ee81ce1 2024-11-23T18:53:01,316 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/5601e81a619a4adca4c9a54bca9c1e86 2024-11-23T18:53:01,321 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/recovered.edits/102.seqid, newMaxSeqId=102, maxSeqId=1 2024-11-23T18:53:01,321 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. 2024-11-23T18:53:01,322 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for a23b45b30de756a61d82ac4bd938eb32: Waiting for close lock at 1732387981275Running coprocessor pre-close hooks at 1732387981275Disabling compacts and flushes for region at 1732387981275Disabling writes for close at 1732387981276 (+1 ms)Obtaining lock to block concurrent updates at 1732387981276Preparing flush snapshotting stores in a23b45b30de756a61d82ac4bd938eb32 at 1732387981276Finished memstore snapshotting TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., syncing WAL and waiting on mvcc, flushsize=dataSize=13663, getHeapSize=14904, getOffHeapSize=0, getCellsCount=13 at 1732387981277 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. at 1732387981278 (+1 ms)Flushing a23b45b30de756a61d82ac4bd938eb32/info: creating writer at 1732387981278Flushing a23b45b30de756a61d82ac4bd938eb32/info: appending metadata at 1732387981284 (+6 ms)Flushing a23b45b30de756a61d82ac4bd938eb32/info: closing flushed file at 1732387981284Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f80e5e1: reopening flushed file at 1732387981298 (+14 ms)Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=0 B/0 for a23b45b30de756a61d82ac4bd938eb32 in 29ms, sequenceid=99, compaction requested=false at 1732387981305 (+7 ms)Writing region close event to WAL at 1732387981318 (+13 ms)Running coprocessor post-close hooks at 1732387981321 (+3 ms)Closed at 1732387981321 2024-11-23T18:53:01,324 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a23b45b30de756a61d82ac4bd938eb32, regionState=CLOSED 2024-11-23T18:53:01,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 because future has completed 2024-11-23T18:53:01,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T18:53:01,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure a23b45b30de756a61d82ac4bd938eb32, server=2eda60943ea1,46533,1732387965202 in 215 msec 2024-11-23T18:53:01,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T18:53:01,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a23b45b30de756a61d82ac4bd938eb32, UNASSIGN in 223 msec 2024-11-23T18:53:01,342 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:01,345 INFO [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 2 storefiles, region=a23b45b30de756a61d82ac4bd938eb32, threads=2 2024-11-23T18:53:01,347 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/d533def941dc4b54abb6d295c753d428 for region: a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,347 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352 for region: a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,359 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/d533def941dc4b54abb6d295c753d428, top=true 2024-11-23T18:53:01,360 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:01,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741845_1021 (size=27) 2024-11-23T18:53:01,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741845_1021 (size=27) 2024-11-23T18:53:01,373 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428 for child: 9052a8baef1d58cfaaabb9d33262b4cc, parent: a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,373 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/d533def941dc4b54abb6d295c753d428 for region: a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741846_1022 (size=27) 2024-11-23T18:53:01,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741846_1022 (size=27) 2024-11-23T18:53:01,379 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352 for region: a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:01,381 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region a23b45b30de756a61d82ac4bd938eb32 Daughter A: [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32] storefiles, Daughter B: [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428] storefiles. 2024-11-23T18:53:01,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741847_1023 (size=71) 2024-11-23T18:53:01,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741847_1023 (size=71) 2024-11-23T18:53:01,390 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741848_1024 (size=71) 2024-11-23T18:53:01,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741848_1024 (size=71) 2024-11-23T18:53:01,405 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:01,414 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/recovered.edits/102.seqid, newMaxSeqId=102, maxSeqId=-1 2024-11-23T18:53:01,416 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/recovered.edits/102.seqid, newMaxSeqId=102, maxSeqId=-1 2024-11-23T18:53:01,418 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732387981418"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732387981418"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732387981418"}]},"ts":"1732387981418"} 2024-11-23T18:53:01,419 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732387981418"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387981418"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732387981418"}]},"ts":"1732387981418"} 2024-11-23T18:53:01,419 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732387981418"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387981418"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732387981418"}]},"ts":"1732387981418"} 2024-11-23T18:53:01,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e4579d7a4e55da8b5d463e229943fc06, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9052a8baef1d58cfaaabb9d33262b4cc, ASSIGN}] 2024-11-23T18:53:01,436 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9052a8baef1d58cfaaabb9d33262b4cc, ASSIGN 2024-11-23T18:53:01,437 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e4579d7a4e55da8b5d463e229943fc06, ASSIGN 2024-11-23T18:53:01,437 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9052a8baef1d58cfaaabb9d33262b4cc, ASSIGN; state=SPLITTING_NEW, location=2eda60943ea1,46533,1732387965202; forceNewPlan=false, retain=false 2024-11-23T18:53:01,437 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e4579d7a4e55da8b5d463e229943fc06, ASSIGN; state=SPLITTING_NEW, location=2eda60943ea1,46533,1732387965202; forceNewPlan=false, retain=false 2024-11-23T18:53:01,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:01,588 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9052a8baef1d58cfaaabb9d33262b4cc, regionState=OPENING, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e4579d7a4e55da8b5d463e229943fc06, regionState=OPENING, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9052a8baef1d58cfaaabb9d33262b4cc, ASSIGN because future has completed 2024-11-23T18:53:01,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9052a8baef1d58cfaaabb9d33262b4cc, server=2eda60943ea1,46533,1732387965202}] 2024-11-23T18:53:01,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e4579d7a4e55da8b5d463e229943fc06, ASSIGN because future has completed 2024-11-23T18:53:01,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4579d7a4e55da8b5d463e229943fc06, server=2eda60943ea1,46533,1732387965202}] 2024-11-23T18:53:01,792 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:01,792 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => e4579d7a4e55da8b5d463e229943fc06, NAME => 'TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.', STARTKEY => '', ENDKEY => 'row0064'} 2024-11-23T18:53:01,793 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,793 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:01,793 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,793 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,795 INFO [StoreOpener-e4579d7a4e55da8b5d463e229943fc06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,797 INFO [StoreOpener-e4579d7a4e55da8b5d463e229943fc06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4579d7a4e55da8b5d463e229943fc06 columnFamilyName info 2024-11-23T18:53:01,797 DEBUG [StoreOpener-e4579d7a4e55da8b5d463e229943fc06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:01,812 DEBUG [StoreOpener-e4579d7a4e55da8b5d463e229943fc06-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-bottom 2024-11-23T18:53:01,812 INFO [StoreOpener-e4579d7a4e55da8b5d463e229943fc06-1 {}] regionserver.HStore(327): Store=e4579d7a4e55da8b5d463e229943fc06/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:53:01,813 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,813 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,814 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,815 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,815 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,817 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,817 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened e4579d7a4e55da8b5d463e229943fc06; next sequenceid=103; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787447, jitterRate=0.0012918710708618164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:53:01,818 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:01,818 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for e4579d7a4e55da8b5d463e229943fc06: Running coprocessor pre-open hook at 1732387981793Writing region info on filesystem at 1732387981793Initializing all the Stores at 1732387981795 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387981795Cleaning up temporary data from old regions at 1732387981815 (+20 ms)Running coprocessor post-open hooks at 1732387981818 (+3 ms)Region opened successfully at 1732387981818 2024-11-23T18:53:01,819 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06., pid=13, masterSystemTime=1732387981787 2024-11-23T18:53:01,819 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store e4579d7a4e55da8b5d463e229943fc06:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:01,819 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,819 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T18:53:01,820 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:01,820 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): e4579d7a4e55da8b5d463e229943fc06/info is initiating minor compaction (all files) 2024-11-23T18:53:01,820 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4579d7a4e55da8b5d463e229943fc06/info in TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:01,820 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-bottom] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/.tmp, totalSize=72.0 K 2024-11-23T18:53:01,821 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32, keycount=32, bloomtype=ROW, size=72.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732387976456 2024-11-23T18:53:01,821 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:01,822 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:01,822 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:01,822 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 9052a8baef1d58cfaaabb9d33262b4cc, NAME => 'TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.', STARTKEY => 'row0064', ENDKEY => ''} 2024-11-23T18:53:01,822 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,822 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:01,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e4579d7a4e55da8b5d463e229943fc06, regionState=OPEN, openSeqNum=103, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,822 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,823 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,824 INFO [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,824 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-23T18:53:01,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T18:53:01,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-11-23T18:53:01,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4579d7a4e55da8b5d463e229943fc06, server=2eda60943ea1,46533,1732387965202 because future has completed 2024-11-23T18:53:01,825 INFO [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9052a8baef1d58cfaaabb9d33262b4cc columnFamilyName info 2024-11-23T18:53:01,825 DEBUG [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:01,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-11-23T18:53:01,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e4579d7a4e55da8b5d463e229943fc06, server=2eda60943ea1,46533,1732387965202 in 191 msec 2024-11-23T18:53:01,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e4579d7a4e55da8b5d463e229943fc06, ASSIGN in 395 msec 2024-11-23T18:53:01,834 DEBUG [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-top 2024-11-23T18:53:01,839 DEBUG [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428 2024-11-23T18:53:01,839 INFO [StoreOpener-9052a8baef1d58cfaaabb9d33262b4cc-1 {}] regionserver.HStore(327): Store=9052a8baef1d58cfaaabb9d33262b4cc/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:53:01,839 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,840 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,841 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,841 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,841 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/d3f60faecd054677b50fa388f8368895 is 193, key is TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc./info:regioninfo/1732387981588/Put/seqid=0 2024-11-23T18:53:01,844 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,844 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4579d7a4e55da8b5d463e229943fc06#info#compaction#39 average throughput is 15.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:01,845 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 9052a8baef1d58cfaaabb9d33262b4cc; next sequenceid=103; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707522, jitterRate=-0.10034017264842987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:53:01,845 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:01,845 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/.tmp/info/6e60827a616e431f9917f58c961d8197 is 1055, key is row0001/info:/1732387976456/Put/seqid=0 2024-11-23T18:53:01,845 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 9052a8baef1d58cfaaabb9d33262b4cc: Running coprocessor pre-open hook at 1732387981823Writing region info on filesystem at 1732387981823Initializing all the Stores at 1732387981823Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387981823Cleaning up temporary data from old regions at 1732387981841 (+18 ms)Running coprocessor post-open hooks at 1732387981845 (+4 ms)Region opened successfully at 1732387981845 2024-11-23T18:53:01,846 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., pid=12, masterSystemTime=1732387981787 2024-11-23T18:53:01,846 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 2 2024-11-23T18:53:01,846 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,846 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T18:53:01,847 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:01,847 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:01,847 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:01,847 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-top, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=90.2 K 2024-11-23T18:53:01,848 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:01,848 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] compactions.Compactor(225): Compacting 582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32, keycount=32, bloomtype=ROW, size=72.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732387976456 2024-11-23T18:53:01,848 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:01,848 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428, keycount=13, bloomtype=ROW, size=18.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732387980611 2024-11-23T18:53:01,849 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9052a8baef1d58cfaaabb9d33262b4cc, regionState=OPEN, openSeqNum=103, regionLocation=2eda60943ea1,46533,1732387965202 2024-11-23T18:53:01,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9052a8baef1d58cfaaabb9d33262b4cc, server=2eda60943ea1,46533,1732387965202 because future has completed 2024-11-23T18:53:01,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741849_1025 (size=9847) 2024-11-23T18:53:01,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741849_1025 (size=9847) 2024-11-23T18:53:01,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/d3f60faecd054677b50fa388f8368895 2024-11-23T18:53:01,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T18:53:01,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 9052a8baef1d58cfaaabb9d33262b4cc, server=2eda60943ea1,46533,1732387965202 in 260 msec 2024-11-23T18:53:01,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741850_1026 (size=71441) 2024-11-23T18:53:01,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741850_1026 (size=71441) 2024-11-23T18:53:01,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-11-23T18:53:01,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9052a8baef1d58cfaaabb9d33262b4cc, ASSIGN in 421 msec 2024-11-23T18:53:01,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a23b45b30de756a61d82ac4bd938eb32, daughterA=e4579d7a4e55da8b5d463e229943fc06, daughterB=9052a8baef1d58cfaaabb9d33262b4cc in 764 msec 2024-11-23T18:53:01,863 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/.tmp/info/6e60827a616e431f9917f58c961d8197 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/6e60827a616e431f9917f58c961d8197 2024-11-23T18:53:01,869 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in e4579d7a4e55da8b5d463e229943fc06/info of e4579d7a4e55da8b5d463e229943fc06 into 6e60827a616e431f9917f58c961d8197(size=69.8 K), total size for store is 69.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:01,869 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4579d7a4e55da8b5d463e229943fc06: 2024-11-23T18:53:01,869 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06., storeName=e4579d7a4e55da8b5d463e229943fc06/info, priority=15, startTime=1732387981819; duration=0sec 2024-11-23T18:53:01,869 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,869 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4579d7a4e55da8b5d463e229943fc06:info 2024-11-23T18:53:01,872 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#41 average throughput is 15.03 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:01,872 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/34d91bf905f6440a8a934b6a2388503b is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741851_1027 (size=20932) 2024-11-23T18:53:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741851_1027 (size=20932) 2024-11-23T18:53:01,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/ns/3605443c4de7406a8304bb05eb22b28c is 43, key is default/ns:d/1732387966260/Put/seqid=0 2024-11-23T18:53:01,884 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/34d91bf905f6440a8a934b6a2388503b as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/34d91bf905f6440a8a934b6a2388503b 2024-11-23T18:53:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741852_1028 (size=5153) 2024-11-23T18:53:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741852_1028 (size=5153) 2024-11-23T18:53:01,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/ns/3605443c4de7406a8304bb05eb22b28c 2024-11-23T18:53:01,891 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 2 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 34d91bf905f6440a8a934b6a2388503b(size=20.4 K), total size for store is 20.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:01,891 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:01,891 INFO [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=14, startTime=1732387981846; duration=0sec 2024-11-23T18:53:01,891 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:01,891 DEBUG [RS:0;2eda60943ea1:46533-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:01,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/table/07b09f7d9ce34675b8201a7ed17c5aa5 is 65, key is TestLogRolling-testLogRolling/table:state/1732387966726/Put/seqid=0 2024-11-23T18:53:01,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741853_1029 (size=5340) 2024-11-23T18:53:01,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741853_1029 (size=5340) 2024-11-23T18:53:01,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/table/07b09f7d9ce34675b8201a7ed17c5aa5 2024-11-23T18:53:01,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/d3f60faecd054677b50fa388f8368895 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/info/d3f60faecd054677b50fa388f8368895 2024-11-23T18:53:01,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/info/d3f60faecd054677b50fa388f8368895, entries=30, sequenceid=17, filesize=9.6 K 2024-11-23T18:53:01,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/ns/3605443c4de7406a8304bb05eb22b28c as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/ns/3605443c4de7406a8304bb05eb22b28c 2024-11-23T18:53:01,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/ns/3605443c4de7406a8304bb05eb22b28c, entries=2, sequenceid=17, filesize=5.0 K 2024-11-23T18:53:01,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/table/07b09f7d9ce34675b8201a7ed17c5aa5 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/table/07b09f7d9ce34675b8201a7ed17c5aa5 2024-11-23T18:53:01,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/table/07b09f7d9ce34675b8201a7ed17c5aa5, entries=2, sequenceid=17, filesize=5.2 K 2024-11-23T18:53:01,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 111ms, sequenceid=17, compaction requested=false 2024-11-23T18:53:01,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-23T18:53:02,361 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:02,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:03,362 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:03,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:04,363 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:04,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:05,365 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:05,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:06,348 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,349 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,349 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,349 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,349 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,350 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,355 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,355 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,358 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,366 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:06,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:06,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T18:53:06,890 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,890 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,890 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,890 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,890 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,891 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,896 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,896 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,897 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:06,900 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:07,367 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:07,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:08,368 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:08,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:09,369 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:09,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:10,370 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:10,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:54148 deadline: 1732388000774, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. is not online on 2eda60943ea1,46533,1732387965202 2024-11-23T18:53:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. is not online on 2eda60943ea1,46533,1732387965202 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T18:53:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32. is not online on 2eda60943ea1,46533,1732387965202 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T18:53:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732387966343.a23b45b30de756a61d82ac4bd938eb32., hostname=2eda60943ea1,46533,1732387965202, seqNum=2 from cache 2024-11-23T18:53:11,371 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:11,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:12,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:12,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:13,401 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:13,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:14,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:14,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:15,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:53:15,403 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:15,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:16,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:16,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:17,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:17,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:18,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:18,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:19,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:19,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:20,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:20,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:21,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:21,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:22,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:22,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:23,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:23,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:24,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:24,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:25,416 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:25,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:26,275 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T18:53:26,275 INFO [master/2eda60943ea1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T18:53:26,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:26,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:27,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:27,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:28,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:28,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:29,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:29,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:30,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:30,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0079', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., hostname=2eda60943ea1,46533,1732387965202, seqNum=103] 2024-11-23T18:53:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:30,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:53:30,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/3bd06ba60dfd493fb24b383c60274027 is 1055, key is row0079/info:/1732388010858/Put/seqid=0 2024-11-23T18:53:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741854_1030 (size=13386) 2024-11-23T18:53:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741854_1030 (size=13386) 2024-11-23T18:53:30,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/3bd06ba60dfd493fb24b383c60274027 2024-11-23T18:53:30,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/3bd06ba60dfd493fb24b383c60274027 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027 2024-11-23T18:53:30,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027, entries=8, sequenceid=114, filesize=13.1 K 2024-11-23T18:53:30,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=10.26 KB/10510 for 9052a8baef1d58cfaaabb9d33262b4cc in 26ms, sequenceid=114, compaction requested=false 2024-11-23T18:53:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:31,190 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-11-23T18:53:31,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:31,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:32,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:32,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:32,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=11.29 KB heapSize=12.37 KB 2024-11-23T18:53:32,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/aa4117666e7d4ea3a59f2dd508e9ccee is 1055, key is row0087/info:/1732388010870/Put/seqid=0 2024-11-23T18:53:32,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741855_1031 (size=16542) 2024-11-23T18:53:32,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741855_1031 (size=16542) 2024-11-23T18:53:32,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.29 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/aa4117666e7d4ea3a59f2dd508e9ccee 2024-11-23T18:53:32,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/aa4117666e7d4ea3a59f2dd508e9ccee as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee 2024-11-23T18:53:32,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee, entries=11, sequenceid=128, filesize=16.2 K 2024-11-23T18:53:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.29 KB/11561, heapSize ~12.35 KB/12648, currentSize=10.26 KB/10510 for 9052a8baef1d58cfaaabb9d33262b4cc in 53ms, sequenceid=128, compaction requested=true 2024-11-23T18:53:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:32,946 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:32,947 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50860 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:32,948 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:32,948 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:32,948 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/34d91bf905f6440a8a934b6a2388503b, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=49.7 K 2024-11-23T18:53:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:32,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-23T18:53:32,948 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34d91bf905f6440a8a934b6a2388503b, keycount=15, bloomtype=ROW, size=20.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732387978601 2024-11-23T18:53:32,949 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd06ba60dfd493fb24b383c60274027, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732388010858 2024-11-23T18:53:32,949 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4117666e7d4ea3a59f2dd508e9ccee, keycount=11, bloomtype=ROW, size=16.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732388010870 2024-11-23T18:53:32,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a4b99d781ca74738a16aca713e992d23 is 1055, key is row0098/info:/1732388012894/Put/seqid=0 2024-11-23T18:53:32,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741856_1032 (size=17606) 2024-11-23T18:53:32,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741856_1032 (size=17606) 2024-11-23T18:53:32,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a4b99d781ca74738a16aca713e992d23 2024-11-23T18:53:32,965 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#47 average throughput is 34.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:32,966 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/df961e287fcc40ed899d1de24293772e is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:32,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741857_1033 (size=41062) 2024-11-23T18:53:32,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741857_1033 (size=41062) 2024-11-23T18:53:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a4b99d781ca74738a16aca713e992d23 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23 2024-11-23T18:53:32,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23, entries=12, sequenceid=143, filesize=17.2 K 2024-11-23T18:53:32,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=10.26 KB/10510 for 9052a8baef1d58cfaaabb9d33262b4cc in 31ms, sequenceid=143, compaction requested=false 2024-11-23T18:53:32,979 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/df961e287fcc40ed899d1de24293772e as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/df961e287fcc40ed899d1de24293772e 2024-11-23T18:53:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:32,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-23T18:53:32,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a99a55c49f0d4c138e3f6c1f3e2032a5 is 1055, key is row0110/info:/1732388012950/Put/seqid=0 2024-11-23T18:53:32,988 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into df961e287fcc40ed899d1de24293772e(size=40.1 K), total size for store is 57.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:32,988 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:32,988 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388012946; duration=0sec 2024-11-23T18:53:32,988 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:32,988 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741858_1034 (size=17606) 2024-11-23T18:53:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741858_1034 (size=17606) 2024-11-23T18:53:32,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a99a55c49f0d4c138e3f6c1f3e2032a5 2024-11-23T18:53:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a99a55c49f0d4c138e3f6c1f3e2032a5 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5 2024-11-23T18:53:33,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5, entries=12, sequenceid=158, filesize=17.2 K 2024-11-23T18:53:33,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=7.18 KB/7357 for 9052a8baef1d58cfaaabb9d33262b4cc in 23ms, sequenceid=158, compaction requested=true 2024-11-23T18:53:33,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:33,005 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:33,006 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 76274 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:33,006 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:33,006 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:33,006 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/df961e287fcc40ed899d1de24293772e, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=74.5 K 2024-11-23T18:53:33,006 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting df961e287fcc40ed899d1de24293772e, keycount=34, bloomtype=ROW, size=40.1 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732387978601 2024-11-23T18:53:33,007 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b99d781ca74738a16aca713e992d23, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732388012894 2024-11-23T18:53:33,007 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting a99a55c49f0d4c138e3f6c1f3e2032a5, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732388012950 2024-11-23T18:53:33,017 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#49 average throughput is 58.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:33,018 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/01885bd7229b474ca2066575dc56d631 is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741859_1035 (size=66504) 2024-11-23T18:53:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741859_1035 (size=66504) 2024-11-23T18:53:33,030 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/01885bd7229b474ca2066575dc56d631 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/01885bd7229b474ca2066575dc56d631 2024-11-23T18:53:33,035 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 01885bd7229b474ca2066575dc56d631(size=64.9 K), total size for store is 64.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:33,036 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:33,036 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388013004; duration=0sec 2024-11-23T18:53:33,036 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:33,036 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:33,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:33,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:34,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:34,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:35,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:53:35,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/583be5b01e534a1eac85cb78853b5d39 is 1055, key is row0122/info:/1732388012983/Put/seqid=0 2024-11-23T18:53:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741860_1036 (size=13394) 2024-11-23T18:53:35,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741860_1036 (size=13394) 2024-11-23T18:53:35,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/583be5b01e534a1eac85cb78853b5d39 2024-11-23T18:53:35,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/583be5b01e534a1eac85cb78853b5d39 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39 2024-11-23T18:53:35,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39, entries=8, sequenceid=171, filesize=13.1 K 2024-11-23T18:53:35,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=11.29 KB/11561 for 9052a8baef1d58cfaaabb9d33262b4cc in 24ms, sequenceid=171, compaction requested=false 2024-11-23T18:53:35,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:35,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-23T18:53:35,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/03e6cc5d95bb41c88a54f77da4da6410 is 1055, key is row0130/info:/1732388015007/Put/seqid=0 2024-11-23T18:53:35,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741861_1037 (size=17606) 2024-11-23T18:53:35,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741861_1037 (size=17606) 2024-11-23T18:53:35,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/03e6cc5d95bb41c88a54f77da4da6410 2024-11-23T18:53:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/03e6cc5d95bb41c88a54f77da4da6410 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410 2024-11-23T18:53:35,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410, entries=12, sequenceid=186, filesize=17.2 K 2024-11-23T18:53:35,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=11.29 KB/11561 for 9052a8baef1d58cfaaabb9d33262b4cc in 25ms, sequenceid=186, compaction requested=true 2024-11-23T18:53:35,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:35,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:35,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:35,055 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:35,056 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 97504 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:35,056 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:35,056 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:35,056 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/01885bd7229b474ca2066575dc56d631, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=95.2 K 2024-11-23T18:53:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:35,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-23T18:53:35,057 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01885bd7229b474ca2066575dc56d631, keycount=58, bloomtype=ROW, size=64.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732387978601 2024-11-23T18:53:35,058 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583be5b01e534a1eac85cb78853b5d39, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732388012983 2024-11-23T18:53:35,058 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e6cc5d95bb41c88a54f77da4da6410, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732388015007 2024-11-23T18:53:35,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/387531d98a7d4f908ea73ac355c53e50 is 1055, key is row0142/info:/1732388015031/Put/seqid=0 2024-11-23T18:53:35,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741862_1038 (size=18675) 2024-11-23T18:53:35,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741862_1038 (size=18675) 2024-11-23T18:53:35,075 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#53 average throughput is 26.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:35,076 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/2b6ace205293487c9645a320bf0c3bca is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741863_1039 (size=87739) 2024-11-23T18:53:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741863_1039 (size=87739) 2024-11-23T18:53:35,089 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/2b6ace205293487c9645a320bf0c3bca as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/2b6ace205293487c9645a320bf0c3bca 2024-11-23T18:53:35,096 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 2b6ace205293487c9645a320bf0c3bca(size=85.7 K), total size for store is 85.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:35,096 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:35,096 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388015055; duration=0sec 2024-11-23T18:53:35,097 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:35,097 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:35,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:35,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/387531d98a7d4f908ea73ac355c53e50 2024-11-23T18:53:35,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/387531d98a7d4f908ea73ac355c53e50 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50 2024-11-23T18:53:35,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50, entries=13, sequenceid=202, filesize=18.2 K 2024-11-23T18:53:35,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=6.16 KB/6306 for 9052a8baef1d58cfaaabb9d33262b4cc in 430ms, sequenceid=202, compaction requested=false 2024-11-23T18:53:35,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:35,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:36,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:36,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:37,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:53:37,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/bb5f1285198e4fa0bd2b7f063ebc664d is 1055, key is row0155/info:/1732388015058/Put/seqid=0 2024-11-23T18:53:37,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741864_1040 (size=13394) 2024-11-23T18:53:37,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741864_1040 (size=13394) 2024-11-23T18:53:37,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/bb5f1285198e4fa0bd2b7f063ebc664d 2024-11-23T18:53:37,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/bb5f1285198e4fa0bd2b7f063ebc664d as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d 2024-11-23T18:53:37,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d, entries=8, sequenceid=214, filesize=13.1 K 2024-11-23T18:53:37,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=11.29 KB/11561 for 9052a8baef1d58cfaaabb9d33262b4cc in 23ms, sequenceid=214, compaction requested=true 2024-11-23T18:53:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:37,105 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:37,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-23T18:53:37,107 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119808 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:37,107 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:37,107 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:37,107 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/2b6ace205293487c9645a320bf0c3bca, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=117 K 2024-11-23T18:53:37,107 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6ace205293487c9645a320bf0c3bca, keycount=78, bloomtype=ROW, size=85.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732387978601 2024-11-23T18:53:37,108 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387531d98a7d4f908ea73ac355c53e50, keycount=13, bloomtype=ROW, size=18.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732388015031 2024-11-23T18:53:37,108 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5f1285198e4fa0bd2b7f063ebc664d, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732388015058 2024-11-23T18:53:37,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/9543ff652ff7451b94c262195321d19e is 1055, key is row0163/info:/1732388017084/Put/seqid=0 2024-11-23T18:53:37,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741865_1041 (size=17606) 2024-11-23T18:53:37,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741865_1041 (size=17606) 2024-11-23T18:53:37,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/9543ff652ff7451b94c262195321d19e 2024-11-23T18:53:37,122 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#56 average throughput is 33.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:37,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/9543ff652ff7451b94c262195321d19e as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e 2024-11-23T18:53:37,123 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/51ad1c4840eb4f94bb8fbbc4495f0014 is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:37,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e, entries=12, sequenceid=229, filesize=17.2 K 2024-11-23T18:53:37,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=9.24 KB/9459 for 9052a8baef1d58cfaaabb9d33262b4cc in 24ms, sequenceid=229, compaction requested=false 2024-11-23T18:53:37,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:37,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=10.26 KB heapSize=11.27 KB 2024-11-23T18:53:37,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741866_1042 (size=109962) 2024-11-23T18:53:37,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741866_1042 (size=109962) 2024-11-23T18:53:37,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1b923c5377b140ad93724f3d2deaae0f is 1055, key is row0175/info:/1732388017107/Put/seqid=0 2024-11-23T18:53:37,144 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/51ad1c4840eb4f94bb8fbbc4495f0014 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/51ad1c4840eb4f94bb8fbbc4495f0014 2024-11-23T18:53:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741867_1043 (size=15500) 2024-11-23T18:53:37,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741867_1043 (size=15500) 2024-11-23T18:53:37,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.26 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1b923c5377b140ad93724f3d2deaae0f 2024-11-23T18:53:37,152 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 51ad1c4840eb4f94bb8fbbc4495f0014(size=107.4 K), total size for store is 124.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:37,152 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:37,152 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388017105; duration=0sec 2024-11-23T18:53:37,152 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:37,152 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1b923c5377b140ad93724f3d2deaae0f as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f 2024-11-23T18:53:37,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f, entries=10, sequenceid=242, filesize=15.1 K 2024-11-23T18:53:37,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.26 KB/10510, heapSize ~11.25 KB/11520, currentSize=8.21 KB/8408 for 9052a8baef1d58cfaaabb9d33262b4cc in 24ms, sequenceid=242, compaction requested=true 2024-11-23T18:53:37,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:37,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:37,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:37,160 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:37,161 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 143068 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:37,161 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:37,161 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:37,161 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/51ad1c4840eb4f94bb8fbbc4495f0014, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=139.7 K 2024-11-23T18:53:37,162 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51ad1c4840eb4f94bb8fbbc4495f0014, keycount=99, bloomtype=ROW, size=107.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732387978601 2024-11-23T18:53:37,162 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9543ff652ff7451b94c262195321d19e, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732388017084 2024-11-23T18:53:37,163 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b923c5377b140ad93724f3d2deaae0f, keycount=10, bloomtype=ROW, size=15.1 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732388017107 2024-11-23T18:53:37,176 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#58 average throughput is 60.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:37,176 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a88075b06843423f9413e8e893ad5b22 is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:37,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741868_1044 (size=133362) 2024-11-23T18:53:37,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741868_1044 (size=133362) 2024-11-23T18:53:37,189 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/a88075b06843423f9413e8e893ad5b22 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a88075b06843423f9413e8e893ad5b22 2024-11-23T18:53:37,195 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into a88075b06843423f9413e8e893ad5b22(size=130.2 K), total size for store is 130.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:37,195 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:37,195 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388017160; duration=0sec 2024-11-23T18:53:37,195 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:37,195 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:37,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:37,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:38,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:38,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:39,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=9.24 KB heapSize=10.16 KB 2024-11-23T18:53:39,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/ea52a4eafe1b4e19ae746ba204468162 is 1055, key is row0185/info:/1732388017137/Put/seqid=0 2024-11-23T18:53:39,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741869_1045 (size=14447) 2024-11-23T18:53:39,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741869_1045 (size=14447) 2024-11-23T18:53:39,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.24 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/ea52a4eafe1b4e19ae746ba204468162 2024-11-23T18:53:39,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/ea52a4eafe1b4e19ae746ba204468162 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162 2024-11-23T18:53:39,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162, entries=9, sequenceid=256, filesize=14.1 K 2024-11-23T18:53:39,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.24 KB/9459, heapSize ~10.15 KB/10392, currentSize=12.32 KB/12612 for 9052a8baef1d58cfaaabb9d33262b4cc in 25ms, sequenceid=256, compaction requested=false 2024-11-23T18:53:39,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:39,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-23T18:53:39,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/dc9969e78aa045cd936da23803670f3c is 1055, key is row0194/info:/1732388019163/Put/seqid=0 2024-11-23T18:53:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741870_1046 (size=18688) 2024-11-23T18:53:39,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741870_1046 (size=18688) 2024-11-23T18:53:39,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/dc9969e78aa045cd936da23803670f3c 2024-11-23T18:53:39,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/dc9969e78aa045cd936da23803670f3c as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c 2024-11-23T18:53:39,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c, entries=13, sequenceid=272, filesize=18.3 K 2024-11-23T18:53:39,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=12.32 KB/12612 for 9052a8baef1d58cfaaabb9d33262b4cc in 22ms, sequenceid=272, compaction requested=true 2024-11-23T18:53:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:39,208 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:39,209 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 166497 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:39,209 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:39,209 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:39,209 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a88075b06843423f9413e8e893ad5b22, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=162.6 K 2024-11-23T18:53:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:39,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=13.34 KB heapSize=14.57 KB 2024-11-23T18:53:39,209 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88075b06843423f9413e8e893ad5b22, keycount=121, bloomtype=ROW, size=130.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732387978601 2024-11-23T18:53:39,210 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea52a4eafe1b4e19ae746ba204468162, keycount=9, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732388017137 2024-11-23T18:53:39,210 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9969e78aa045cd936da23803670f3c, keycount=13, bloomtype=ROW, size=18.3 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732388019163 2024-11-23T18:53:39,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1e1fac9fdbfc4a9e8cd00ac702738f62 is 1055, key is row0207/info:/1732388019186/Put/seqid=0 2024-11-23T18:53:39,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741871_1047 (size=18688) 2024-11-23T18:53:39,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741871_1047 (size=18688) 2024-11-23T18:53:39,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.34 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1e1fac9fdbfc4a9e8cd00ac702738f62 2024-11-23T18:53:39,225 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#62 average throughput is 35.83 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:39,226 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/8e311520fe504dc58db37a50e75c9d5a is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:39,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741872_1048 (size=156719) 2024-11-23T18:53:39,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741872_1048 (size=156719) 2024-11-23T18:53:39,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1e1fac9fdbfc4a9e8cd00ac702738f62 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62 2024-11-23T18:53:39,234 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/8e311520fe504dc58db37a50e75c9d5a as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/8e311520fe504dc58db37a50e75c9d5a 2024-11-23T18:53:39,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62, entries=13, sequenceid=288, filesize=18.3 K 2024-11-23T18:53:39,240 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 8e311520fe504dc58db37a50e75c9d5a(size=153.0 K), total size for store is 171.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:39,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.34 KB/13663, heapSize ~14.55 KB/14904, currentSize=5.13 KB/5255 for 9052a8baef1d58cfaaabb9d33262b4cc in 31ms, sequenceid=288, compaction requested=false 2024-11-23T18:53:39,240 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:39,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:39,240 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388019207; duration=0sec 2024-11-23T18:53:39,240 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:39,240 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:39,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:39,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:40,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:40,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:41,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=8.21 KB heapSize=9.06 KB 2024-11-23T18:53:41,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/b952043e67b449afbec2be1a8997e7c7 is 1055, key is row0220/info:/1732388019210/Put/seqid=0 2024-11-23T18:53:41,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741873_1049 (size=13402) 2024-11-23T18:53:41,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741873_1049 (size=13402) 2024-11-23T18:53:41,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.21 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/b952043e67b449afbec2be1a8997e7c7 2024-11-23T18:53:41,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/b952043e67b449afbec2be1a8997e7c7 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7 2024-11-23T18:53:41,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7, entries=8, sequenceid=300, filesize=13.1 K 2024-11-23T18:53:41,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.21 KB/8408, heapSize ~9.05 KB/9264, currentSize=11.29 KB/11561 for 9052a8baef1d58cfaaabb9d33262b4cc in 23ms, sequenceid=300, compaction requested=true 2024-11-23T18:53:41,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:41,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:41,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:41,256 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:41,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=12.32 KB heapSize=13.47 KB 2024-11-23T18:53:41,257 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 188809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:41,257 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:41,257 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:41,257 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/8e311520fe504dc58db37a50e75c9d5a, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=184.4 K 2024-11-23T18:53:41,258 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e311520fe504dc58db37a50e75c9d5a, keycount=143, bloomtype=ROW, size=153.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732387978601 2024-11-23T18:53:41,258 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e1fac9fdbfc4a9e8cd00ac702738f62, keycount=13, bloomtype=ROW, size=18.3 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732388019186 2024-11-23T18:53:41,259 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting b952043e67b449afbec2be1a8997e7c7, keycount=8, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732388019210 2024-11-23T18:53:41,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/86159f36f5984428a7b2011d1bd60037 is 1055, key is row0228/info:/1732388021234/Put/seqid=0 2024-11-23T18:53:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741874_1050 (size=17618) 2024-11-23T18:53:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741874_1050 (size=17618) 2024-11-23T18:53:41,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.32 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/86159f36f5984428a7b2011d1bd60037 2024-11-23T18:53:41,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/86159f36f5984428a7b2011d1bd60037 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037 2024-11-23T18:53:41,273 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#65 average throughput is 54.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:41,274 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/e92b959ac4a44d3f800f6f3c1e1bdcd1 is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741875_1051 (size=178959) 2024-11-23T18:53:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741875_1051 (size=178959) 2024-11-23T18:53:41,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037, entries=12, sequenceid=315, filesize=17.2 K 2024-11-23T18:53:41,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.32 KB/12612, heapSize ~13.45 KB/13776, currentSize=10.26 KB/10510 for 9052a8baef1d58cfaaabb9d33262b4cc in 22ms, sequenceid=315, compaction requested=false 2024-11-23T18:53:41,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:41,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=11.29 KB heapSize=12.37 KB 2024-11-23T18:53:41,284 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/e92b959ac4a44d3f800f6f3c1e1bdcd1 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/e92b959ac4a44d3f800f6f3c1e1bdcd1 2024-11-23T18:53:41,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/21dab9ab47f248ef8b1dd53a6b43cbcc is 1055, key is row0240/info:/1732388021257/Put/seqid=0 2024-11-23T18:53:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741876_1052 (size=16564) 2024-11-23T18:53:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741876_1052 (size=16564) 2024-11-23T18:53:41,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.29 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/21dab9ab47f248ef8b1dd53a6b43cbcc 2024-11-23T18:53:41,291 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into e92b959ac4a44d3f800f6f3c1e1bdcd1(size=174.8 K), total size for store is 192.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:41,291 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:41,291 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388021256; duration=0sec 2024-11-23T18:53:41,291 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:41,291 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/21dab9ab47f248ef8b1dd53a6b43cbcc as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc 2024-11-23T18:53:41,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc, entries=11, sequenceid=329, filesize=16.2 K 2024-11-23T18:53:41,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.29 KB/11561, heapSize ~12.35 KB/12648, currentSize=6.16 KB/6306 for 9052a8baef1d58cfaaabb9d33262b4cc in 21ms, sequenceid=329, compaction requested=true 2024-11-23T18:53:41,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:41,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9052a8baef1d58cfaaabb9d33262b4cc:info, priority=-2147483648, current under compaction store size is 1 2024-11-23T18:53:41,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:41,301 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T18:53:41,302 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 213141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T18:53:41,303 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1541): 9052a8baef1d58cfaaabb9d33262b4cc/info is initiating minor compaction (all files) 2024-11-23T18:53:41,303 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9052a8baef1d58cfaaabb9d33262b4cc/info in TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:41,303 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/e92b959ac4a44d3f800f6f3c1e1bdcd1, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc] into tmpdir=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp, totalSize=208.1 K 2024-11-23T18:53:41,303 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting e92b959ac4a44d3f800f6f3c1e1bdcd1, keycount=164, bloomtype=ROW, size=174.8 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732387978601 2024-11-23T18:53:41,303 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86159f36f5984428a7b2011d1bd60037, keycount=12, bloomtype=ROW, size=17.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732388021234 2024-11-23T18:53:41,304 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21dab9ab47f248ef8b1dd53a6b43cbcc, keycount=11, bloomtype=ROW, size=16.2 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732388021257 2024-11-23T18:53:41,315 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9052a8baef1d58cfaaabb9d33262b4cc#info#compaction#67 average throughput is 62.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:53:41,316 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1f5c5ce2967a40a6a9cdcb10ccbe495c is 1055, key is row0064/info:/1732387978601/Put/seqid=0 2024-11-23T18:53:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741877_1053 (size=203307) 2024-11-23T18:53:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741877_1053 (size=203307) 2024-11-23T18:53:41,324 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/1f5c5ce2967a40a6a9cdcb10ccbe495c as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1f5c5ce2967a40a6a9cdcb10ccbe495c 2024-11-23T18:53:41,330 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9052a8baef1d58cfaaabb9d33262b4cc/info of 9052a8baef1d58cfaaabb9d33262b4cc into 1f5c5ce2967a40a6a9cdcb10ccbe495c(size=198.5 K), total size for store is 198.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:53:41,330 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:41,330 INFO [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc., storeName=9052a8baef1d58cfaaabb9d33262b4cc/info, priority=13, startTime=1732388021301; duration=0sec 2024-11-23T18:53:41,330 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T18:53:41,330 DEBUG [RS:0;2eda60943ea1:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9052a8baef1d58cfaaabb9d33262b4cc:info 2024-11-23T18:53:41,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:41,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:42,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:42,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:43,293 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-23T18:53:43,313 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023294, exclude list is [], retry=0 2024-11-23T18:53:43,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46545,DS-35dd9149-e2ad-4314-ba5c-dcebffabe180,DISK] 2024-11-23T18:53:43,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41159,DS-e098b697-be36-48fe-997e-a766f9421635,DISK] 2024-11-23T18:53:43,317 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732387965820 with entries=316, filesize=303.40 KB; new WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023294 2024-11-23T18:53:43,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:33185:33185)] 2024-11-23T18:53:43,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732387965820 is not closed yet, will try archiving it next time 2024-11-23T18:53:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741833_1009 (size=310694) 2024-11-23T18:53:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741833_1009 (size=310694) 2024-11-23T18:53:43,322 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e4579d7a4e55da8b5d463e229943fc06: 2024-11-23T18:53:43,322 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-11-23T18:53:43,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/1e1d240cedf3473cac57693329ef46dd is 193, key is TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc./info:regioninfo/1732387981848/Put/seqid=0 2024-11-23T18:53:43,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741879_1055 (size=6223) 2024-11-23T18:53:43,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741879_1055 (size=6223) 2024-11-23T18:53:43,332 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/1e1d240cedf3473cac57693329ef46dd 2024-11-23T18:53:43,337 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/.tmp/info/1e1d240cedf3473cac57693329ef46dd as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/info/1e1d240cedf3473cac57693329ef46dd 2024-11-23T18:53:43,342 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/info/1e1d240cedf3473cac57693329ef46dd, entries=5, sequenceid=21, filesize=6.1 K 2024-11-23T18:53:43,343 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false 2024-11-23T18:53:43,343 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-23T18:53:43,343 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9052a8baef1d58cfaaabb9d33262b4cc 1/1 column families, dataSize=6.16 KB heapSize=6.86 KB 2024-11-23T18:53:43,347 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/7641e1d24cbe4e52b85fed8ec9a48ba6 is 1055, key is row0251/info:/1732388021281/Put/seqid=0 2024-11-23T18:53:43,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741880_1056 (size=11286) 2024-11-23T18:53:43,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741880_1056 (size=11286) 2024-11-23T18:53:43,352 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.16 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/7641e1d24cbe4e52b85fed8ec9a48ba6 2024-11-23T18:53:43,357 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/.tmp/info/7641e1d24cbe4e52b85fed8ec9a48ba6 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/7641e1d24cbe4e52b85fed8ec9a48ba6 2024-11-23T18:53:43,363 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/7641e1d24cbe4e52b85fed8ec9a48ba6, entries=6, sequenceid=340, filesize=11.0 K 2024-11-23T18:53:43,364 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~6.16 KB/6306, heapSize ~6.84 KB/7008, currentSize=0 B/0 for 9052a8baef1d58cfaaabb9d33262b4cc in 21ms, sequenceid=340, compaction requested=false 2024-11-23T18:53:43,364 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9052a8baef1d58cfaaabb9d33262b4cc: 2024-11-23T18:53:43,379 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023364, exclude list is [], retry=0 2024-11-23T18:53:43,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46545,DS-35dd9149-e2ad-4314-ba5c-dcebffabe180,DISK] 2024-11-23T18:53:43,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41159,DS-e098b697-be36-48fe-997e-a766f9421635,DISK] 2024-11-23T18:53:43,382 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023294 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023364 2024-11-23T18:53:43,382 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33185:33185),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-23T18:53:43,383 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023294 is not closed yet, will try archiving it next time 2024-11-23T18:53:43,383 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732387965820 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs/2eda60943ea1%2C46533%2C1732387965202.1732387965820 2024-11-23T18:53:43,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:53:43,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741878_1054 (size=731) 2024-11-23T18:53:43,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741878_1054 (size=731) 2024-11-23T18:53:43,384 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/WALs/2eda60943ea1,46533,1732387965202/2eda60943ea1%2C46533%2C1732387965202.1732388023294 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs/2eda60943ea1%2C46533%2C1732387965202.1732388023294 2024-11-23T18:53:43,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:43,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:53:43,484 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:53:43,484 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:43,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:43,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:43,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:53:43,485 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:53:43,485 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1857602714, stopped=false 2024-11-23T18:53:43,485 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,34751,1732387965032 2024-11-23T18:53:43,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:43,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:43,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:43,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:43,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:53:43,567 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:53:43,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:43,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:43,568 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:43,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:43,569 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,46533,1732387965202' ***** 2024-11-23T18:53:43,570 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:53:43,570 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:53:43,570 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:53:43,570 INFO [RS:0;2eda60943ea1:46533 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(3091): Received CLOSE for e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(3091): Received CLOSE for 9052a8baef1d58cfaaabb9d33262b4cc 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,46533,1732387965202 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:53:43,571 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e4579d7a4e55da8b5d463e229943fc06, disabling compactions & flushes 2024-11-23T18:53:43,571 INFO [RS:0;2eda60943ea1:46533 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:46533. 2024-11-23T18:53:43,571 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:43,571 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:43,571 DEBUG [RS:0;2eda60943ea1:46533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:43,571 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. after waiting 0 ms 2024-11-23T18:53:43,571 DEBUG [RS:0;2eda60943ea1:46533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:43,571 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:43,572 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:53:43,572 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:53:43,572 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:53:43,572 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:53:43,572 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-23T18:53:43,572 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1325): Online Regions={e4579d7a4e55da8b5d463e229943fc06=TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06., 1588230740=hbase:meta,,1.1588230740, 9052a8baef1d58cfaaabb9d33262b4cc=TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.} 2024-11-23T18:53:43,572 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-bottom] to archive 2024-11-23T18:53:43,572 DEBUG [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9052a8baef1d58cfaaabb9d33262b4cc, e4579d7a4e55da8b5d463e229943fc06 2024-11-23T18:53:43,572 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:53:43,572 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:53:43,573 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:53:43,573 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:53:43,573 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:53:43,574 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:53:43,576 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:43,576 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2eda60943ea1:34751 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-23T18:53:43,576 WARN [StoreCloser-TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-23T18:53:43,578 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-23T18:53:43,579 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:53:43,579 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:53:43,579 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732388023572Running coprocessor pre-close hooks at 1732388023572Disabling compacts and flushes for region at 1732388023572Disabling writes for close at 1732388023573 (+1 ms)Writing region close event to WAL at 1732388023575 (+2 ms)Running coprocessor post-close hooks at 1732388023578 (+3 ms)Closed at 1732388023579 (+1 ms) 2024-11-23T18:53:43,579 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:53:43,580 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/e4579d7a4e55da8b5d463e229943fc06/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=102 2024-11-23T18:53:43,580 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:43,580 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e4579d7a4e55da8b5d463e229943fc06: Waiting for close lock at 1732388023571Running coprocessor pre-close hooks at 1732388023571Disabling compacts and flushes for region at 1732388023571Disabling writes for close at 1732388023571Writing region close event to WAL at 1732388023577 (+6 ms)Running coprocessor post-close hooks at 1732388023580 (+3 ms)Closed at 1732388023580 2024-11-23T18:53:43,580 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732387981095.e4579d7a4e55da8b5d463e229943fc06. 2024-11-23T18:53:43,580 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9052a8baef1d58cfaaabb9d33262b4cc, disabling compactions & flushes 2024-11-23T18:53:43,580 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:43,581 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:43,581 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. after waiting 0 ms 2024-11-23T18:53:43,581 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:43,581 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32->hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/a23b45b30de756a61d82ac4bd938eb32/info/582dadf957b5455e820e7df589ca7352-top, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/34d91bf905f6440a8a934b6a2388503b, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/df961e287fcc40ed899d1de24293772e, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/01885bd7229b474ca2066575dc56d631, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/2b6ace205293487c9645a320bf0c3bca, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/51ad1c4840eb4f94bb8fbbc4495f0014, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a88075b06843423f9413e8e893ad5b22, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/8e311520fe504dc58db37a50e75c9d5a, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/e92b959ac4a44d3f800f6f3c1e1bdcd1, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc] to archive 2024-11-23T18:53:43,582 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:53:43,584 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/582dadf957b5455e820e7df589ca7352.a23b45b30de756a61d82ac4bd938eb32 2024-11-23T18:53:43,585 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/34d91bf905f6440a8a934b6a2388503b to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/34d91bf905f6440a8a934b6a2388503b 2024-11-23T18:53:43,586 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/TestLogRolling-testLogRolling=a23b45b30de756a61d82ac4bd938eb32-d533def941dc4b54abb6d295c753d428 2024-11-23T18:53:43,588 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/3bd06ba60dfd493fb24b383c60274027 2024-11-23T18:53:43,589 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/df961e287fcc40ed899d1de24293772e to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/df961e287fcc40ed899d1de24293772e 2024-11-23T18:53:43,590 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/aa4117666e7d4ea3a59f2dd508e9ccee 2024-11-23T18:53:43,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:43,591 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a4b99d781ca74738a16aca713e992d23 2024-11-23T18:53:43,592 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/01885bd7229b474ca2066575dc56d631 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/01885bd7229b474ca2066575dc56d631 2024-11-23T18:53:43,593 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a99a55c49f0d4c138e3f6c1f3e2032a5 2024-11-23T18:53:43,594 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/583be5b01e534a1eac85cb78853b5d39 2024-11-23T18:53:43,595 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/2b6ace205293487c9645a320bf0c3bca to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/2b6ace205293487c9645a320bf0c3bca 2024-11-23T18:53:43,596 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/03e6cc5d95bb41c88a54f77da4da6410 2024-11-23T18:53:43,597 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/387531d98a7d4f908ea73ac355c53e50 2024-11-23T18:53:43,597 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/51ad1c4840eb4f94bb8fbbc4495f0014 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/51ad1c4840eb4f94bb8fbbc4495f0014 2024-11-23T18:53:43,598 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/bb5f1285198e4fa0bd2b7f063ebc664d 2024-11-23T18:53:43,599 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/9543ff652ff7451b94c262195321d19e 2024-11-23T18:53:43,600 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a88075b06843423f9413e8e893ad5b22 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/a88075b06843423f9413e8e893ad5b22 2024-11-23T18:53:43,601 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1b923c5377b140ad93724f3d2deaae0f 2024-11-23T18:53:43,602 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/ea52a4eafe1b4e19ae746ba204468162 2024-11-23T18:53:43,602 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/8e311520fe504dc58db37a50e75c9d5a to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/8e311520fe504dc58db37a50e75c9d5a 2024-11-23T18:53:43,603 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/dc9969e78aa045cd936da23803670f3c 2024-11-23T18:53:43,604 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/1e1fac9fdbfc4a9e8cd00ac702738f62 2024-11-23T18:53:43,605 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/e92b959ac4a44d3f800f6f3c1e1bdcd1 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/e92b959ac4a44d3f800f6f3c1e1bdcd1 2024-11-23T18:53:43,606 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/b952043e67b449afbec2be1a8997e7c7 2024-11-23T18:53:43,607 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037 to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/86159f36f5984428a7b2011d1bd60037 2024-11-23T18:53:43,608 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc to hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/archive/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/info/21dab9ab47f248ef8b1dd53a6b43cbcc 2024-11-23T18:53:43,608 WARN [StoreCloser-TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [34d91bf905f6440a8a934b6a2388503b=20932, 3bd06ba60dfd493fb24b383c60274027=13386, df961e287fcc40ed899d1de24293772e=41062, aa4117666e7d4ea3a59f2dd508e9ccee=16542, a4b99d781ca74738a16aca713e992d23=17606, 01885bd7229b474ca2066575dc56d631=66504, a99a55c49f0d4c138e3f6c1f3e2032a5=17606, 583be5b01e534a1eac85cb78853b5d39=13394, 2b6ace205293487c9645a320bf0c3bca=87739, 03e6cc5d95bb41c88a54f77da4da6410=17606, 387531d98a7d4f908ea73ac355c53e50=18675, 51ad1c4840eb4f94bb8fbbc4495f0014=109962, bb5f1285198e4fa0bd2b7f063ebc664d=13394, 9543ff652ff7451b94c262195321d19e=17606, a88075b06843423f9413e8e893ad5b22=133362, 1b923c5377b140ad93724f3d2deaae0f=15500, ea52a4eafe1b4e19ae746ba204468162=14447, 8e311520fe504dc58db37a50e75c9d5a=156719, dc9969e78aa045cd936da23803670f3c=18688, 1e1fac9fdbfc4a9e8cd00ac702738f62=18688, e92b959ac4a44d3f800f6f3c1e1bdcd1=178959, b952043e67b449afbec2be1a8997e7c7=13402, 86159f36f5984428a7b2011d1bd60037=17618, 21dab9ab47f248ef8b1dd53a6b43cbcc=16564] 2024-11-23T18:53:43,611 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/data/default/TestLogRolling-testLogRolling/9052a8baef1d58cfaaabb9d33262b4cc/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=102 2024-11-23T18:53:43,611 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:43,612 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9052a8baef1d58cfaaabb9d33262b4cc: Waiting for close lock at 1732388023580Running coprocessor pre-close hooks at 1732388023580Disabling compacts and flushes for region at 1732388023580Disabling writes for close at 1732388023581 (+1 ms)Writing region close event to WAL at 1732388023608 (+27 ms)Running coprocessor post-close hooks at 1732388023611 (+3 ms)Closed at 1732388023611 2024-11-23T18:53:43,612 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0064,1732387981095.9052a8baef1d58cfaaabb9d33262b4cc. 2024-11-23T18:53:43,688 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T18:53:43,688 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T18:53:43,691 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:53:43,773 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,46533,1732387965202; all regions closed. 2024-11-23T18:53:43,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741834_1010 (size=8107) 2024-11-23T18:53:43,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741834_1010 (size=8107) 2024-11-23T18:53:43,783 DEBUG [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs 2024-11-23T18:53:43,783 INFO [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C46533%2C1732387965202.meta:.meta(num 1732387966171) 2024-11-23T18:53:43,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741881_1057 (size=778) 2024-11-23T18:53:43,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741881_1057 (size=778) 2024-11-23T18:53:43,789 DEBUG [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/oldWALs 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C46533%2C1732387965202:(num 1732388023364) 2024-11-23T18:53:43,790 DEBUG [RS:0;2eda60943ea1:46533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:53:43,790 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:53:43,790 INFO [RS:0;2eda60943ea1:46533 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46533 2024-11-23T18:53:43,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:53:43,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,46533,1732387965202 2024-11-23T18:53:43,801 INFO [RS:0;2eda60943ea1:46533 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:53:43,802 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,46533,1732387965202] 2024-11-23T18:53:43,822 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,46533,1732387965202 already deleted, retry=false 2024-11-23T18:53:43,822 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,46533,1732387965202 expired; onlineServers=0 2024-11-23T18:53:43,822 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,34751,1732387965032' ***** 2024-11-23T18:53:43,822 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:53:43,822 INFO [M:0;2eda60943ea1:34751 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:53:43,822 INFO [M:0;2eda60943ea1:34751 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:53:43,823 DEBUG [M:0;2eda60943ea1:34751 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:53:43,823 DEBUG [M:0;2eda60943ea1:34751 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:53:43,823 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:53:43,823 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387965543 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387965543,5,FailOnTimeoutGroup] 2024-11-23T18:53:43,823 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387965543 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387965543,5,FailOnTimeoutGroup] 2024-11-23T18:53:43,823 INFO [M:0;2eda60943ea1:34751 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:53:43,824 INFO [M:0;2eda60943ea1:34751 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:53:43,824 DEBUG [M:0;2eda60943ea1:34751 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:53:43,824 INFO [M:0;2eda60943ea1:34751 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:53:43,824 INFO [M:0;2eda60943ea1:34751 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:53:43,825 INFO [M:0;2eda60943ea1:34751 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:53:43,825 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:53:43,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:53:43,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:43,833 DEBUG [M:0;2eda60943ea1:34751 {}] zookeeper.ZKUtil(347): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:53:43,833 WARN [M:0;2eda60943ea1:34751 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:53:43,835 INFO [M:0;2eda60943ea1:34751 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/.lastflushedseqids 2024-11-23T18:53:43,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741882_1058 (size=228) 2024-11-23T18:53:43,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741882_1058 (size=228) 2024-11-23T18:53:43,848 INFO [M:0;2eda60943ea1:34751 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:53:43,848 INFO [M:0;2eda60943ea1:34751 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:53:43,848 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:53:43,848 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:43,848 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:43,848 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:53:43,849 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:43,849 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.41 KB heapSize=63.33 KB 2024-11-23T18:53:43,862 DEBUG [M:0;2eda60943ea1:34751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3ac1f1d2a9432b97b6be07727b7c91 is 82, key is hbase:meta,,1/info:regioninfo/1732387966207/Put/seqid=0 2024-11-23T18:53:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741883_1059 (size=5672) 2024-11-23T18:53:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741883_1059 (size=5672) 2024-11-23T18:53:43,867 INFO [M:0;2eda60943ea1:34751 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3ac1f1d2a9432b97b6be07727b7c91 2024-11-23T18:53:43,885 DEBUG [M:0;2eda60943ea1:34751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c606ec98d9e0414591a10c960f5aa270 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387966731/Put/seqid=0 2024-11-23T18:53:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741884_1060 (size=7089) 2024-11-23T18:53:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741884_1060 (size=7089) 2024-11-23T18:53:43,891 INFO [M:0;2eda60943ea1:34751 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.80 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c606ec98d9e0414591a10c960f5aa270 2024-11-23T18:53:43,895 INFO [M:0;2eda60943ea1:34751 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c606ec98d9e0414591a10c960f5aa270 2024-11-23T18:53:43,908 DEBUG [M:0;2eda60943ea1:34751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87b29e3496c54cab935e9dbfc8d2c41d is 69, key is 2eda60943ea1,46533,1732387965202/rs:state/1732387965669/Put/seqid=0 2024-11-23T18:53:43,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:43,912 INFO [RS:0;2eda60943ea1:46533 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:53:43,912 INFO [RS:0;2eda60943ea1:46533 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,46533,1732387965202; zookeeper connection closed. 2024-11-23T18:53:43,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x101690a84060001, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:43,912 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@282b8509 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@282b8509 2024-11-23T18:53:43,912 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T18:53:43,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741885_1061 (size=5156) 2024-11-23T18:53:43,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741885_1061 (size=5156) 2024-11-23T18:53:43,913 INFO [M:0;2eda60943ea1:34751 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87b29e3496c54cab935e9dbfc8d2c41d 2024-11-23T18:53:43,929 DEBUG [M:0;2eda60943ea1:34751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e54200981fe746f78ac2b78e89dd4c0c is 52, key is load_balancer_on/state:d/1732387966338/Put/seqid=0 2024-11-23T18:53:43,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741886_1062 (size=5056) 2024-11-23T18:53:43,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741886_1062 (size=5056) 2024-11-23T18:53:43,934 INFO [M:0;2eda60943ea1:34751 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e54200981fe746f78ac2b78e89dd4c0c 2024-11-23T18:53:43,939 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3ac1f1d2a9432b97b6be07727b7c91 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb3ac1f1d2a9432b97b6be07727b7c91 2024-11-23T18:53:43,944 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb3ac1f1d2a9432b97b6be07727b7c91, entries=8, sequenceid=125, filesize=5.5 K 2024-11-23T18:53:43,945 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c606ec98d9e0414591a10c960f5aa270 as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c606ec98d9e0414591a10c960f5aa270 2024-11-23T18:53:43,949 INFO [M:0;2eda60943ea1:34751 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c606ec98d9e0414591a10c960f5aa270 2024-11-23T18:53:43,949 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c606ec98d9e0414591a10c960f5aa270, entries=13, sequenceid=125, filesize=6.9 K 2024-11-23T18:53:43,950 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87b29e3496c54cab935e9dbfc8d2c41d as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/87b29e3496c54cab935e9dbfc8d2c41d 2024-11-23T18:53:43,955 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/87b29e3496c54cab935e9dbfc8d2c41d, entries=1, sequenceid=125, filesize=5.0 K 2024-11-23T18:53:43,955 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e54200981fe746f78ac2b78e89dd4c0c as hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e54200981fe746f78ac2b78e89dd4c0c 2024-11-23T18:53:43,960 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39149/user/jenkins/test-data/155dfccf-536a-af4e-626e-9c5717205b40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e54200981fe746f78ac2b78e89dd4c0c, entries=1, sequenceid=125, filesize=4.9 K 2024-11-23T18:53:43,961 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=125, compaction requested=false 2024-11-23T18:53:43,962 INFO [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:43,962 DEBUG [M:0;2eda60943ea1:34751 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732388023848Disabling compacts and flushes for region at 1732388023848Disabling writes for close at 1732388023849 (+1 ms)Obtaining lock to block concurrent updates at 1732388023849Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732388023849Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52639, getHeapSize=64784, getOffHeapSize=0, getCellsCount=148 at 1732388023849Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732388023849Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732388023849Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732388023861 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732388023862 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732388023871 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732388023885 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732388023885Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732388023895 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732388023908 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732388023908Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732388023917 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732388023929 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732388023929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79db4864: reopening flushed file at 1732388023938 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18ce6cf4: reopening flushed file at 1732388023944 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31bba101: reopening flushed file at 1732388023950 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6acfd143: reopening flushed file at 1732388023955 (+5 ms)Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=125, compaction requested=false at 1732388023961 (+6 ms)Writing region close event to WAL at 1732388023962 (+1 ms)Closed at 1732388023962 2024-11-23T18:53:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46545 is added to blk_1073741830_1006 (size=61308) 2024-11-23T18:53:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41159 is added to blk_1073741830_1006 (size=61308) 2024-11-23T18:53:43,965 INFO [M:0;2eda60943ea1:34751 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:53:43,965 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:53:43,965 INFO [M:0;2eda60943ea1:34751 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34751 2024-11-23T18:53:43,965 INFO [M:0;2eda60943ea1:34751 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:53:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:44,076 INFO [M:0;2eda60943ea1:34751 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:53:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34751-0x101690a84060000, quorum=127.0.0.1:56959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:44,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e9a7ffb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:44,113 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b485209{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:44,113 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:44,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@dc32d49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:44,114 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@88cc783{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:44,116 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:53:44,117 WARN [BP-1758811751-172.17.0.2-1732387962631 heartbeating to localhost/127.0.0.1:39149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:53:44,117 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:53:44,117 WARN [BP-1758811751-172.17.0.2-1732387962631 heartbeating to localhost/127.0.0.1:39149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1758811751-172.17.0.2-1732387962631 (Datanode Uuid 69490fd6-ea36-4691-ab17-44fd903fda7c) service to localhost/127.0.0.1:39149 2024-11-23T18:53:44,118 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data3/current/BP-1758811751-172.17.0.2-1732387962631 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:44,119 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data4/current/BP-1758811751-172.17.0.2-1732387962631 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:44,119 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:53:44,122 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@483a949e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:44,122 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@76e3af92{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:44,122 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:44,122 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b6547b7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:44,122 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@325aed0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:44,124 WARN [BP-1758811751-172.17.0.2-1732387962631 heartbeating to localhost/127.0.0.1:39149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:53:44,124 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:53:44,124 WARN [BP-1758811751-172.17.0.2-1732387962631 heartbeating to localhost/127.0.0.1:39149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1758811751-172.17.0.2-1732387962631 (Datanode Uuid d48fff1c-97df-4257-9676-5652a32cabe0) service to localhost/127.0.0.1:39149 2024-11-23T18:53:44,124 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:53:44,125 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data1/current/BP-1758811751-172.17.0.2-1732387962631 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:44,125 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/cluster_c0fdc23e-78b3-43b8-62f2-6faf56c44ed4/data/data2/current/BP-1758811751-172.17.0.2-1732387962631 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:44,125 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:53:44,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ace8d8a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:53:44,131 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22ac2cd9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:44,131 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:44,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e9b2284{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:44,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@758d42f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:44,137 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:53:44,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T18:53:44,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRolling Thread=110 (was 97) - Thread LEAK? -, OpenFileDescriptor=458 (was 428) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=79 (was 86), ProcessCount=11 (was 11), AvailableMemoryMB=8570 (was 8736) 2024-11-23T18:53:44,190 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncLogRolling#testLogRollOnNothingWritten Thread=110, OpenFileDescriptor=458, MaxFileDescriptor=1048576, SystemLoadAverage=79, ProcessCount=11, AvailableMemoryMB=8569 2024-11-23T18:53:44,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.log.dir so I do NOT create it in target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5a6ee918-9c1b-6a75-c767-fabd8f574e20/hadoop.tmp.dir so I do NOT create it in target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da, deleteOnExit=true 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/test.cache.data in system properties and HBase conf 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:53:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:53:44,192 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:53:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:53:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:53:44,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:44,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:44,713 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:53:44,717 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:53:44,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:53:44,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:53:44,720 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:53:44,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:53:44,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469c0328{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:53:44,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@314da9e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:53:44,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d5c391e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/java.io.tmpdir/jetty-localhost-38643-hadoop-hdfs-3_4_1-tests_jar-_-any-3140358850008813617/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:53:44,814 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1db11f13{HTTP/1.1, (http/1.1)}{localhost:38643} 2024-11-23T18:53:44,814 INFO [Time-limited test {}] server.Server(415): Started @218964ms 2024-11-23T18:53:45,156 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:53:45,159 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:53:45,160 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:53:45,160 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:53:45,160 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:53:45,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57bc2473{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:53:45,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21e874ca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:53:45,254 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e0d45ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/java.io.tmpdir/jetty-localhost-37367-hadoop-hdfs-3_4_1-tests_jar-_-any-8162759262068656023/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:45,254 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11aa3304{HTTP/1.1, (http/1.1)}{localhost:37367} 2024-11-23T18:53:45,254 INFO [Time-limited test {}] server.Server(415): Started @219404ms 2024-11-23T18:53:45,255 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:53:45,282 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:53:45,285 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:53:45,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:53:45,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:53:45,286 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:53:45,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6af0ddfd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:53:45,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71a6807e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:53:45,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12fc2a37{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/java.io.tmpdir/jetty-localhost-34253-hadoop-hdfs-3_4_1-tests_jar-_-any-6328685219542153649/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:45,378 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32bf705{HTTP/1.1, (http/1.1)}{localhost:34253} 2024-11-23T18:53:45,378 INFO [Time-limited test {}] server.Server(415): Started @219527ms 2024-11-23T18:53:45,379 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:53:45,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:45,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:46,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:46,484 WARN [Thread-1626 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data1/current/BP-764246846-172.17.0.2-1732388024218/current, will proceed with Du for space computation calculation, 2024-11-23T18:53:46,485 WARN [Thread-1627 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data2/current/BP-764246846-172.17.0.2-1732388024218/current, will proceed with Du for space computation calculation, 2024-11-23T18:53:46,502 WARN [Thread-1590 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:53:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b9ada72d64ce1e1 with lease ID 0x43782cf94a3a673f: Processing first storage report for DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19 from datanode DatanodeRegistration(127.0.0.1:33953, datanodeUuid=f5168d60-6a4b-45e0-bf05-a63b8405743c, infoPort=42723, infoSecurePort=0, ipcPort=34361, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218) 2024-11-23T18:53:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b9ada72d64ce1e1 with lease ID 0x43782cf94a3a673f: from storage DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19 node DatanodeRegistration(127.0.0.1:33953, datanodeUuid=f5168d60-6a4b-45e0-bf05-a63b8405743c, infoPort=42723, infoSecurePort=0, ipcPort=34361, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:53:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b9ada72d64ce1e1 with lease ID 0x43782cf94a3a673f: Processing first storage report for DS-d2c0b03f-57ff-49c7-b58d-78b06dd1a1c5 from datanode DatanodeRegistration(127.0.0.1:33953, datanodeUuid=f5168d60-6a4b-45e0-bf05-a63b8405743c, infoPort=42723, infoSecurePort=0, ipcPort=34361, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218) 2024-11-23T18:53:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b9ada72d64ce1e1 with lease ID 0x43782cf94a3a673f: from storage DS-d2c0b03f-57ff-49c7-b58d-78b06dd1a1c5 node DatanodeRegistration(127.0.0.1:33953, datanodeUuid=f5168d60-6a4b-45e0-bf05-a63b8405743c, infoPort=42723, infoSecurePort=0, ipcPort=34361, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:53:46,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:46,624 WARN [Thread-1637 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data3/current/BP-764246846-172.17.0.2-1732388024218/current, will proceed with Du for space computation calculation, 2024-11-23T18:53:46,625 WARN [Thread-1638 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data4/current/BP-764246846-172.17.0.2-1732388024218/current, will proceed with Du for space computation calculation, 2024-11-23T18:53:46,642 WARN [Thread-1613 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:53:46,643 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86ebe920ddb6148 with lease ID 0x43782cf94a3a6740: Processing first storage report for DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a from datanode DatanodeRegistration(127.0.0.1:46353, datanodeUuid=7b38e634-d26a-444a-ae92-8cc87b03cb53, infoPort=41467, infoSecurePort=0, ipcPort=40443, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218) 2024-11-23T18:53:46,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86ebe920ddb6148 with lease ID 0x43782cf94a3a6740: from storage DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a node DatanodeRegistration(127.0.0.1:46353, datanodeUuid=7b38e634-d26a-444a-ae92-8cc87b03cb53, infoPort=41467, infoSecurePort=0, ipcPort=40443, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:53:46,644 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86ebe920ddb6148 with lease ID 0x43782cf94a3a6740: Processing first storage report for DS-788a5022-390f-4971-8484-fa0ec9d98e67 from datanode DatanodeRegistration(127.0.0.1:46353, datanodeUuid=7b38e634-d26a-444a-ae92-8cc87b03cb53, infoPort=41467, infoSecurePort=0, ipcPort=40443, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218) 2024-11-23T18:53:46,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86ebe920ddb6148 with lease ID 0x43782cf94a3a6740: from storage DS-788a5022-390f-4971-8484-fa0ec9d98e67 node DatanodeRegistration(127.0.0.1:46353, datanodeUuid=7b38e634-d26a-444a-ae92-8cc87b03cb53, infoPort=41467, infoSecurePort=0, ipcPort=40443, storageInfo=lv=-57;cid=testClusterID;nsid=638989560;c=1732388024218), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:53:46,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1 2024-11-23T18:53:46,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/zookeeper_0, clientPort=61082, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:53:46,728 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61082 2024-11-23T18:53:46,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,730 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:53:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:53:46,738 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46 with version=8 2024-11-23T18:53:46,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34337/user/jenkins/test-data/b2325eb6-15ab-bebe-15c3-828170450a0e/hbase-staging 2024-11-23T18:53:46,740 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:53:46,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:53:46,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:53:46,741 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:53:46,741 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:53:46,741 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41083 2024-11-23T18:53:46,743 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41083 connecting to ZooKeeper ensemble=127.0.0.1:61082 2024-11-23T18:53:46,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:410830x0, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:53:46,810 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41083-0x101690b75110000 connected 2024-11-23T18:53:46,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:46,908 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46, hbase.cluster.distributed=false 2024-11-23T18:53:46,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:53:46,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41083 2024-11-23T18:53:46,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41083 2024-11-23T18:53:46,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41083 2024-11-23T18:53:46,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41083 2024-11-23T18:53:46,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41083 2024-11-23T18:53:46,926 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:53:46,926 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:53:46,927 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:53:46,927 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43619 2024-11-23T18:53:46,928 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43619 connecting to ZooKeeper ensemble=127.0.0.1:61082 2024-11-23T18:53:46,929 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,930 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:46,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436190x0, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:53:46,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43619-0x101690b75110001 connected 2024-11-23T18:53:46,945 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:46,945 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:53:46,946 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:53:46,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:53:46,947 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:53:46,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43619 2024-11-23T18:53:46,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43619 2024-11-23T18:53:46,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43619 2024-11-23T18:53:46,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43619 2024-11-23T18:53:46,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43619 2024-11-23T18:53:46,960 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:41083 2024-11-23T18:53:46,961 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,41083,1732388026740 2024-11-23T18:53:46,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:53:46,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:53:46,969 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,41083,1732388026740 2024-11-23T18:53:46,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:53:46,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:46,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:46,980 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:53:46,980 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,41083,1732388026740 from backup master directory 2024-11-23T18:53:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,41083,1732388026740 2024-11-23T18:53:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:53:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:53:46,990 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:53:46,990 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,41083,1732388026740 2024-11-23T18:53:46,995 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/hbase.id] with ID: 8e052018-51f4-46da-aa63-e439293886dc 2024-11-23T18:53:46,996 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/.tmp/hbase.id 2024-11-23T18:53:47,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:53:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:53:47,005 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/.tmp/hbase.id]:[hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/hbase.id] 2024-11-23T18:53:47,017 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:47,017 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:53:47,019 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T18:53:47,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:53:47,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:53:47,040 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:53:47,041 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:53:47,042 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:53:47,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:53:47,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:53:47,049 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store 2024-11-23T18:53:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:53:47,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:53:47,055 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:47,055 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:47,055 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732388027055Disabling compacts and flushes for region at 1732388027055Disabling writes for close at 1732388027055Writing region close event to WAL at 1732388027055Closed at 1732388027055 2024-11-23T18:53:47,056 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/.initializing 2024-11-23T18:53:47,056 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/WALs/2eda60943ea1,41083,1732388026740 2024-11-23T18:53:47,057 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:53:47,058 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C41083%2C1732388026740, suffix=, logDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/WALs/2eda60943ea1,41083,1732388026740, archiveDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/oldWALs, maxLogs=10 2024-11-23T18:53:47,069 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/WALs/2eda60943ea1,41083,1732388026740/2eda60943ea1%2C41083%2C1732388026740.1732388027058, exclude list is [], retry=0 2024-11-23T18:53:47,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33953,DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19,DISK] 2024-11-23T18:53:47,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46353,DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a,DISK] 2024-11-23T18:53:47,073 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/WALs/2eda60943ea1,41083,1732388026740/2eda60943ea1%2C41083%2C1732388026740.1732388027058 2024-11-23T18:53:47,074 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42723:42723),(127.0.0.1/127.0.0.1:41467:41467)] 2024-11-23T18:53:47,074 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:53:47,074 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:47,074 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,074 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:53:47,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:53:47,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:53:47,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:53:47,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:53:47,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:53:47,081 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:53:47,082 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,083 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,083 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,084 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,084 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,084 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:53:47,085 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:53:47,087 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:53:47,088 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799653, jitterRate=0.016812339425086975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:53:47,088 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732388027074Initializing all the Stores at 1732388027075 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027075Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732388027075Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732388027075Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732388027075Cleaning up temporary data from old regions at 1732388027084 (+9 ms)Region opened successfully at 1732388027088 (+4 ms) 2024-11-23T18:53:47,088 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:53:47,091 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af666cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:53:47,092 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:53:47,092 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:53:47,092 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:53:47,093 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:53:47,093 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T18:53:47,093 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T18:53:47,093 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:53:47,095 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:53:47,096 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:53:47,106 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:53:47,106 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:53:47,107 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:53:47,116 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:53:47,116 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:53:47,117 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:53:47,127 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:53:47,128 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:53:47,137 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:53:47,140 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:53:47,148 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:53:47,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:47,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:47,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,160 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,41083,1732388026740, sessionid=0x101690b75110000, setting cluster-up flag (Was=false) 2024-11-23T18:53:47,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,211 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:53:47,212 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,41083,1732388026740 2024-11-23T18:53:47,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,264 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:53:47,265 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,41083,1732388026740 2024-11-23T18:53:47,267 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:53:47,268 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:53:47,269 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:53:47,269 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:53:47,269 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,41083,1732388026740 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:53:47,271 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,272 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732388057272 2024-11-23T18:53:47,272 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:53:47,272 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:53:47,272 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:53:47,273 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:53:47,273 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:53:47,273 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:53:47,274 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732388027274,5,FailOnTimeoutGroup] 2024-11-23T18:53:47,274 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732388027274,5,FailOnTimeoutGroup] 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,274 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,275 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,275 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:53:47,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:53:47,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:53:47,281 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:53:47,281 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46 2024-11-23T18:53:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:53:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:53:47,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:47,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:53:47,289 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:53:47,289 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:53:47,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:53:47,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:53:47,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:53:47,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:53:47,293 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:53:47,293 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:53:47,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740 2024-11-23T18:53:47,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740 2024-11-23T18:53:47,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:53:47,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:53:47,295 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:53:47,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:53:47,297 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:53:47,298 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=704554, jitterRate=-0.1041141152381897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:53:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732388027287Initializing all the Stores at 1732388027287Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027288 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027288Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732388027288Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027288Cleaning up temporary data from old regions at 1732388027295 (+7 ms)Region opened successfully at 1732388027298 (+3 ms) 2024-11-23T18:53:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:53:47,298 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:53:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:53:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:53:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:53:47,299 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:53:47,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732388027298Disabling compacts and flushes for region at 1732388027298Disabling writes for close at 1732388027298Writing region close event to WAL at 1732388027299 (+1 ms)Closed at 1732388027299 2024-11-23T18:53:47,300 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:53:47,300 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:53:47,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:53:47,301 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:53:47,302 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:53:47,352 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(746): ClusterId : 8e052018-51f4-46da-aa63-e439293886dc 2024-11-23T18:53:47,352 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:53:47,368 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:53:47,368 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:53:47,382 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:53:47,383 DEBUG [RS:0;2eda60943ea1:43619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599334f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:53:47,399 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:43619 2024-11-23T18:53:47,399 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:53:47,399 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:53:47,399 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:53:47,400 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,41083,1732388026740 with port=43619, startcode=1732388026926 2024-11-23T18:53:47,400 DEBUG [RS:0;2eda60943ea1:43619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:53:47,401 INFO [HMaster-EventLoopGroup-11-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:53:47,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41083 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41083 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,403 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46 2024-11-23T18:53:47,403 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39633 2024-11-23T18:53:47,403 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:53:47,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:53:47,411 DEBUG [RS:0;2eda60943ea1:43619 {}] zookeeper.ZKUtil(111): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,411 WARN [RS:0;2eda60943ea1:43619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:53:47,412 INFO [RS:0;2eda60943ea1:43619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:53:47,412 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,412 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,43619,1732388026926] 2024-11-23T18:53:47,415 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:53:47,417 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:53:47,417 INFO [RS:0;2eda60943ea1:43619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:53:47,417 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,417 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:53:47,418 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:53:47,418 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,418 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,418 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,418 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,418 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,418 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:53:47,419 DEBUG [RS:0;2eda60943ea1:43619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:53:47,419 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,419 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,420 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,420 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,420 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,420 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43619,1732388026926-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:53:47,435 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:53:47,435 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,43619,1732388026926-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,436 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,436 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.Replication(171): 2eda60943ea1,43619,1732388026926 started 2024-11-23T18:53:47,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:47,448 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,449 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,43619,1732388026926, RpcServer on 2eda60943ea1/172.17.0.2:43619, sessionid=0x101690b75110001 2024-11-23T18:53:47,449 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:53:47,449 DEBUG [RS:0;2eda60943ea1:43619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,449 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43619,1732388026926' 2024-11-23T18:53:47,449 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:53:47,449 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,43619,1732388026926' 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:53:47,450 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:53:47,451 DEBUG [RS:0;2eda60943ea1:43619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:53:47,451 INFO [RS:0;2eda60943ea1:43619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:53:47,451 INFO [RS:0;2eda60943ea1:43619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:53:47,452 WARN [2eda60943ea1:41083 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:53:47,551 INFO [RS:0;2eda60943ea1:43619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:53:47,554 INFO [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43619%2C1732388026926, suffix=, logDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926, archiveDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs, maxLogs=32 2024-11-23T18:53:47,572 DEBUG [RS:0;2eda60943ea1:43619 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926/2eda60943ea1%2C43619%2C1732388026926.1732388027555, exclude list is [], retry=0 2024-11-23T18:53:47,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46353,DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a,DISK] 2024-11-23T18:53:47,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33953,DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19,DISK] 2024-11-23T18:53:47,576 INFO [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926/2eda60943ea1%2C43619%2C1732388026926.1732388027555 2024-11-23T18:53:47,577 DEBUG [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41467:41467),(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-23T18:53:47,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:47,702 DEBUG [2eda60943ea1:41083 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T18:53:47,703 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,706 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43619,1732388026926, state=OPENING 2024-11-23T18:53:47,716 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:53:47,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:47,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:53:47,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:53:47,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43619,1732388026926}] 2024-11-23T18:53:47,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:53:47,884 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:53:47,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60507, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:53:47,888 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:53:47,888 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:53:47,888 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:53:47,890 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C43619%2C1732388026926.meta, suffix=.meta, logDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926, archiveDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs, maxLogs=32 2024-11-23T18:53:47,901 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926/2eda60943ea1%2C43619%2C1732388026926.meta.1732388027890.meta, exclude list is [], retry=0 2024-11-23T18:53:47,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46353,DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a,DISK] 2024-11-23T18:53:47,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33953,DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19,DISK] 2024-11-23T18:53:47,905 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/2eda60943ea1,43619,1732388026926/2eda60943ea1%2C43619%2C1732388026926.meta.1732388027890.meta 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41467:41467),(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:53:47,906 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:53:47,906 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:53:47,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:53:47,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:53:47,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:53:47,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:53:47,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:53:47,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:53:47,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:53:47,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:53:47,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:53:47,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:53:47,913 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:53:47,914 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740 2024-11-23T18:53:47,915 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740 2024-11-23T18:53:47,916 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:53:47,916 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:53:47,916 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:53:47,918 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:53:47,918 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=798369, jitterRate=0.015179648995399475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:53:47,918 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:53:47,919 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732388027906Writing region info on filesystem at 1732388027906Initializing all the Stores at 1732388027907 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027907Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027908 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732388027908Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732388027908Cleaning up temporary data from old regions at 1732388027916 (+8 ms)Running coprocessor post-open hooks at 1732388027919 (+3 ms)Region opened successfully at 1732388027919 2024-11-23T18:53:47,920 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732388027883 2024-11-23T18:53:47,922 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:53:47,922 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:53:47,922 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,923 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,43619,1732388026926, state=OPEN 2024-11-23T18:53:47,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:53:47,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:53:47,957 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,43619,1732388026926 2024-11-23T18:53:47,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:53:47,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:53:47,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:53:47,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,43619,1732388026926 in 228 msec 2024-11-23T18:53:47,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:53:47,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 659 msec 2024-11-23T18:53:47,962 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:53:47,962 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:53:47,963 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:53:47,963 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43619,1732388026926, seqNum=-1] 2024-11-23T18:53:47,963 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:53:47,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:53:47,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 700 msec 2024-11-23T18:53:47,969 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732388027969, completionTime=-1 2024-11-23T18:53:47,969 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T18:53:47,969 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732388087971 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732388147971 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:41083, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,971 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:47,973 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:53:47,975 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.985sec 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:53:47,976 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:53:47,978 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:53:47,978 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:53:47,978 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41083,1732388026740-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:53:48,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e5cdf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:53:48,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,41083,-1 for getting cluster id 2024-11-23T18:53:48,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:53:48,053 DEBUG [HMaster-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8e052018-51f4-46da-aa63-e439293886dc' 2024-11-23T18:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8e052018-51f4-46da-aa63-e439293886dc" 2024-11-23T18:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1993fc12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:53:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,41083,-1] 2024-11-23T18:53:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:53:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,054 INFO [HMaster-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:53:48,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4d9590, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:53:48,055 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:53:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,43619,1732388026926, seqNum=-1] 2024-11-23T18:53:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:53:48,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:53:48,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,41083,1732388026740 2024-11-23T18:53:48,059 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:53:48,061 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-23T18:53:48,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:53:48,063 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:53:48,064 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1, archiveDir=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs, maxLogs=32 2024-11-23T18:53:48,076 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028065, exclude list is [], retry=0 2024-11-23T18:53:48,079 DEBUG [AsyncFSWAL-13-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46353,DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a,DISK] 2024-11-23T18:53:48,079 DEBUG [AsyncFSWAL-13-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33953,DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19,DISK] 2024-11-23T18:53:48,081 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028065 2024-11-23T18:53:48,081 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41467:41467),(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-23T18:53:48,094 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028081, exclude list is [], retry=0 2024-11-23T18:53:48,096 DEBUG [AsyncFSWAL-13-4 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33953,DS-2e3264a0-b37c-4b58-9db8-c1fdc78b8d19,DISK] 2024-11-23T18:53:48,096 DEBUG [AsyncFSWAL-13-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46353,DS-abb73084-f2c3-41b2-a32d-4f205e95ce8a,DISK] 2024-11-23T18:53:48,098 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028065 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028081 2024-11-23T18:53:48,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41467:41467),(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-23T18:53:48,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028065 is not closed yet, will try archiving it next time 2024-11-23T18:53:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741835_1011 (size=93) 2024-11-23T18:53:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741835_1011 (size=93) 2024-11-23T18:53:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741836_1012 (size=93) 2024-11-23T18:53:48,103 WARN [Close-WAL-Writer-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028081 not finished, retry = 0 2024-11-23T18:53:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741836_1012 (size=93) 2024-11-23T18:53:48,103 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/WALs/test.com,8080,1/test.com%2C8080%2C1.1732388028065 to hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs/test.com%2C8080%2C1.1732388028065 2024-11-23T18:53:48,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs 2024-11-23T18:53:48,207 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL test.com%2C8080%2C1:(num 1732388028081) 2024-11-23T18:53:48,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:53:48,207 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:53:48,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:48,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:53:48,208 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:53:48,209 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1403135850, stopped=false 2024-11-23T18:53:48,209 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,41083,1732388026740 2024-11-23T18:53:48,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:48,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:53:48,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:48,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:48,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:53:48,232 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:53:48,232 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:48,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,233 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:48,233 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,43619,1732388026926' ***** 2024-11-23T18:53:48,233 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:53:48,233 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:53:48,233 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,43619,1732388026926 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:53:48,233 INFO [RS:0;2eda60943ea1:43619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:43619. 2024-11-23T18:53:48,233 DEBUG [RS:0;2eda60943ea1:43619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:53:48,233 DEBUG [RS:0;2eda60943ea1:43619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,234 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:53:48,234 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:53:48,234 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:53:48,234 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:53:48,234 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T18:53:48,234 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T18:53:48,234 DEBUG [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T18:53:48,234 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:53:48,234 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:53:48,234 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:53:48,234 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:53:48,234 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:53:48,234 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-23T18:53:48,248 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/.tmp/ns/b563f7e56d664c7fb19dc8058772c4c6 is 43, key is default/ns:d/1732388027965/Put/seqid=0 2024-11-23T18:53:48,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741837_1013 (size=5153) 2024-11-23T18:53:48,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741837_1013 (size=5153) 2024-11-23T18:53:48,254 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/.tmp/ns/b563f7e56d664c7fb19dc8058772c4c6 2024-11-23T18:53:48,259 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/.tmp/ns/b563f7e56d664c7fb19dc8058772c4c6 as hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/ns/b563f7e56d664c7fb19dc8058772c4c6 2024-11-23T18:53:48,264 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/ns/b563f7e56d664c7fb19dc8058772c4c6, entries=2, sequenceid=6, filesize=5.0 K 2024-11-23T18:53:48,265 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false 2024-11-23T18:53:48,266 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:53:48,270 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T18:53:48,270 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:53:48,271 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:53:48,271 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732388028234Running coprocessor pre-close hooks at 1732388028234Disabling compacts and flushes for region at 1732388028234Disabling writes for close at 1732388028234Obtaining lock to block concurrent updates at 1732388028234Preparing flush snapshotting stores in 1588230740 at 1732388028234Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732388028235 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732388028235Flushing 1588230740/ns: creating writer at 1732388028235Flushing 1588230740/ns: appending metadata at 1732388028248 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732388028248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60c1778e: reopening flushed file at 1732388028258 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1732388028265 (+7 ms)Writing region close event to WAL at 1732388028267 (+2 ms)Running coprocessor post-close hooks at 1732388028270 (+3 ms)Closed at 1732388028271 (+1 ms) 2024-11-23T18:53:48,271 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:53:48,424 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T18:53:48,424 INFO [regionserver/2eda60943ea1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T18:53:48,434 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,43619,1732388026926; all regions closed. 2024-11-23T18:53:48,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741834_1010 (size=1152) 2024-11-23T18:53:48,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741834_1010 (size=1152) 2024-11-23T18:53:48,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.1732387878846 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:48,445 DEBUG [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs 2024-11-23T18:53:48,445 INFO [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43619%2C1732388026926.meta:.meta(num 1732388027890) 2024-11-23T18:53:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741833_1009 (size=93) 2024-11-23T18:53:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741833_1009 (size=93) 2024-11-23T18:53:48,450 DEBUG [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/oldWALs 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C43619%2C1732388026926:(num 1732388027555) 2024-11-23T18:53:48,450 DEBUG [RS:0;2eda60943ea1:43619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:53:48,450 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:53:48,450 INFO [RS:0;2eda60943ea1:43619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43619 2024-11-23T18:53:48,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:53:48,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,43619,1732388026926 2024-11-23T18:53:48,460 INFO [RS:0;2eda60943ea1:43619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:53:48,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,43619,1732388026926] 2024-11-23T18:53:48,481 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,43619,1732388026926 already deleted, retry=false 2024-11-23T18:53:48,481 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,43619,1732388026926 expired; onlineServers=0 2024-11-23T18:53:48,481 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,41083,1732388026740' ***** 2024-11-23T18:53:48,481 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:53:48,481 INFO [M:0;2eda60943ea1:41083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:53:48,481 INFO [M:0;2eda60943ea1:41083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:53:48,481 DEBUG [M:0;2eda60943ea1:41083 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:53:48,481 DEBUG [M:0;2eda60943ea1:41083 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:53:48,481 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:53:48,481 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732388027274 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732388027274,5,FailOnTimeoutGroup] 2024-11-23T18:53:48,481 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732388027274 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732388027274,5,FailOnTimeoutGroup] 2024-11-23T18:53:48,482 INFO [M:0;2eda60943ea1:41083 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:53:48,482 INFO [M:0;2eda60943ea1:41083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:53:48,482 DEBUG [M:0;2eda60943ea1:41083 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:53:48,482 INFO [M:0;2eda60943ea1:41083 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:53:48,482 INFO [M:0;2eda60943ea1:41083 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:53:48,482 INFO [M:0;2eda60943ea1:41083 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:53:48,482 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:53:48,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:53:48,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:53:48,492 DEBUG [M:0;2eda60943ea1:41083 {}] zookeeper.ZKUtil(347): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:53:48,492 WARN [M:0;2eda60943ea1:41083 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:53:48,492 INFO [M:0;2eda60943ea1:41083 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/.lastflushedseqids 2024-11-23T18:53:48,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741838_1014 (size=99) 2024-11-23T18:53:48,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741838_1014 (size=99) 2024-11-23T18:53:48,497 INFO [M:0;2eda60943ea1:41083 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:53:48,497 INFO [M:0;2eda60943ea1:41083 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:53:48,497 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:53:48,497 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:48,497 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:48,497 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:53:48,497 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:48,497 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-23T18:53:48,510 DEBUG [M:0;2eda60943ea1:41083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d68c179084f54097b90e059f631452fc is 82, key is hbase:meta,,1/info:regioninfo/1732388027922/Put/seqid=0 2024-11-23T18:53:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741839_1015 (size=5672) 2024-11-23T18:53:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741839_1015 (size=5672) 2024-11-23T18:53:48,515 INFO [M:0;2eda60943ea1:41083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d68c179084f54097b90e059f631452fc 2024-11-23T18:53:48,533 DEBUG [M:0;2eda60943ea1:41083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c57bf4dcdb7c4d30a344be96c3ea4a71 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732388027968/Put/seqid=0 2024-11-23T18:53:48,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741840_1016 (size=5275) 2024-11-23T18:53:48,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741840_1016 (size=5275) 2024-11-23T18:53:48,538 INFO [M:0;2eda60943ea1:41083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c57bf4dcdb7c4d30a344be96c3ea4a71 2024-11-23T18:53:48,559 DEBUG [M:0;2eda60943ea1:41083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ab491a6802f4dcf8ea798f77e6ae1ec is 69, key is 2eda60943ea1,43619,1732388026926/rs:state/1732388027402/Put/seqid=0 2024-11-23T18:53:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741841_1017 (size=5156) 2024-11-23T18:53:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741841_1017 (size=5156) 2024-11-23T18:53:48,565 INFO [M:0;2eda60943ea1:41083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ab491a6802f4dcf8ea798f77e6ae1ec 2024-11-23T18:53:48,571 INFO [RS:0;2eda60943ea1:43619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:53:48,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:48,571 INFO [RS:0;2eda60943ea1:43619 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,43619,1732388026926; zookeeper connection closed. 2024-11-23T18:53:48,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43619-0x101690b75110001, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:48,571 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7635ae06 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7635ae06 2024-11-23T18:53:48,571 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T18:53:48,579 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,579 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,583 DEBUG [M:0;2eda60943ea1:41083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/89dd385d7f694d6fa34ab8b8fa9088c2 is 52, key is load_balancer_on/state:d/1732388028061/Put/seqid=0 2024-11-23T18:53:48,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741842_1018 (size=5056) 2024-11-23T18:53:48,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741842_1018 (size=5056) 2024-11-23T18:53:48,588 INFO [M:0;2eda60943ea1:41083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/89dd385d7f694d6fa34ab8b8fa9088c2 2024-11-23T18:53:48,593 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d68c179084f54097b90e059f631452fc as hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d68c179084f54097b90e059f631452fc 2024-11-23T18:53:48,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:38745/user/jenkins/test-data/4dabe7f4-ec40-676a-f683-2ec1c090896f/WALs/2eda60943ea1,34197,1732387877820/2eda60943ea1%2C34197%2C1732387877820.meta.1732387879318.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-23T18:53:48,598 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,598 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d68c179084f54097b90e059f631452fc, entries=8, sequenceid=29, filesize=5.5 K 2024-11-23T18:53:48,598 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,599 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c57bf4dcdb7c4d30a344be96c3ea4a71 as hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c57bf4dcdb7c4d30a344be96c3ea4a71 2024-11-23T18:53:48,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,600 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,603 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c57bf4dcdb7c4d30a344be96c3ea4a71, entries=3, sequenceid=29, filesize=5.2 K 2024-11-23T18:53:48,604 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ab491a6802f4dcf8ea798f77e6ae1ec as hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ab491a6802f4dcf8ea798f77e6ae1ec 2024-11-23T18:53:48,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,608 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ab491a6802f4dcf8ea798f77e6ae1ec, entries=1, sequenceid=29, filesize=5.0 K 2024-11-23T18:53:48,608 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T18:53:48,609 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/89dd385d7f694d6fa34ab8b8fa9088c2 as hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/89dd385d7f694d6fa34ab8b8fa9088c2 2024-11-23T18:53:48,613 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39633/user/jenkins/test-data/6170492c-c983-7a00-f1ae-ae3b76915a46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/89dd385d7f694d6fa34ab8b8fa9088c2, entries=1, sequenceid=29, filesize=4.9 K 2024-11-23T18:53:48,614 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=29, compaction requested=false 2024-11-23T18:53:48,616 INFO [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:53:48,616 DEBUG [M:0;2eda60943ea1:41083 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732388028497Disabling compacts and flushes for region at 1732388028497Disabling writes for close at 1732388028497Obtaining lock to block concurrent updates at 1732388028497Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732388028497Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732388028497Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732388028498 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732388028498Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732388028510 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732388028510Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732388028520 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732388028533 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732388028533Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732388028543 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732388028559 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732388028559Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732388028569 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732388028583 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732388028583Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28eaa6d6: reopening flushed file at 1732388028593 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54e60e40: reopening flushed file at 1732388028598 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c335b25: reopening flushed file at 1732388028603 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dff52ae: reopening flushed file at 1732388028608 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=29, compaction requested=false at 1732388028614 (+6 ms)Writing region close event to WAL at 1732388028615 (+1 ms)Closed at 1732388028615 2024-11-23T18:53:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46353 is added to blk_1073741830_1006 (size=10311) 2024-11-23T18:53:48,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33953 is added to blk_1073741830_1006 (size=10311) 2024-11-23T18:53:48,618 INFO [M:0;2eda60943ea1:41083 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:53:48,618 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:53:48,618 INFO [M:0;2eda60943ea1:41083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41083 2024-11-23T18:53:48,618 INFO [M:0;2eda60943ea1:41083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:53:48,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:48,729 INFO [M:0;2eda60943ea1:41083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:53:48,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41083-0x101690b75110000, quorum=127.0.0.1:61082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:53:48,731 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12fc2a37{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:48,731 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32bf705{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:48,731 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:48,731 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71a6807e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:48,731 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6af0ddfd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:48,732 WARN [BP-764246846-172.17.0.2-1732388024218 heartbeating to localhost/127.0.0.1:39633 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:53:48,732 WARN [BP-764246846-172.17.0.2-1732388024218 heartbeating to localhost/127.0.0.1:39633 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-764246846-172.17.0.2-1732388024218 (Datanode Uuid 7b38e634-d26a-444a-ae92-8cc87b03cb53) service to localhost/127.0.0.1:39633 2024-11-23T18:53:48,733 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data3/current/BP-764246846-172.17.0.2-1732388024218 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:48,733 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data4/current/BP-764246846-172.17.0.2-1732388024218 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:48,733 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:53:48,733 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:53:48,733 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:53:48,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e0d45ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:53:48,735 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11aa3304{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:48,735 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:48,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21e874ca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:48,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57bc2473{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:48,736 WARN [BP-764246846-172.17.0.2-1732388024218 heartbeating to localhost/127.0.0.1:39633 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:53:48,736 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:53:48,736 WARN [BP-764246846-172.17.0.2-1732388024218 heartbeating to localhost/127.0.0.1:39633 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-764246846-172.17.0.2-1732388024218 (Datanode Uuid f5168d60-6a4b-45e0-bf05-a63b8405743c) service to localhost/127.0.0.1:39633 2024-11-23T18:53:48,736 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:53:48,737 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data1/current/BP-764246846-172.17.0.2-1732388024218 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:48,737 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/cluster_e1300941-ce71-013d-5235-f039b167a4da/data/data2/current/BP-764246846-172.17.0.2-1732388024218 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:53:48,737 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:53:48,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d5c391e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:53:48,742 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1db11f13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:53:48,742 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:53:48,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@314da9e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:53:48,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@469c0328{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79b8f3d4-ae34-8b28-d61b-34a54dc5c3e1/hadoop.log.dir/,STOPPED} 2024-11-23T18:53:48,747 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:53:48,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T18:53:48,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncLogRolling#testLogRollOnNothingWritten Thread=132 (was 110) - Thread LEAK? -, OpenFileDescriptor=513 (was 458) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=81 (was 79) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8515 (was 8569)